Skip to main content

subc_daemon/
supervise.rs

1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5    collections::{HashMap, HashSet, VecDeque},
6    error::Error,
7    fmt, io,
8    path::PathBuf,
9    process::{ExitStatus, Stdio},
10    sync::{Arc, Mutex, OnceLock},
11    time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22    manifest::{SelfSignalKind, SignalAnchor},
23    session::{
24        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25        MODULE_CONTROL_OP_HEALTH_CHECK,
26    },
27    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30    process::{Child, Command},
31    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32    task::JoinHandle,
33    time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38    child_roster::ChildRoster,
39    daemon_config::{
40        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41    },
42    forwarding::{
43        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44        ModuleDrainTarget, PendingModuleControlRpc,
45    },
46    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47    registry::{ConnectionId, RegistrationEndReason, RegistryError},
48    stderr_tail::{
49        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50        StderrTailSnapshot,
51    },
52    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53    Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75mod host_health;
76use host_health::HostHealth;
77
78#[cfg(all(test, unix))]
79mod slow_retry_tests;
80
81/// Command-line flag used by supervised modules to find subc.
82///
83/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
84/// The path points at the TCP+key connection file; it is not an ambient signal and
85/// is never inherited by standalone children.
86pub const SUBC_ARG: &str = "--subc";
87
88const DEFAULT_MAX_RESTARTS: u32 = 3;
89const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
90const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
91/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
92/// to contain a real crash loop (which respawns in seconds) and short enough
93/// that unrelated crashes hours apart never accumulate into a permanent stop.
94const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
95/// How long a drain waits for already-dispatched requests to finalize before
96/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
97/// not RPC-scale: the original 2s value silently cut nearly every real tool
98/// call at the fence, making the wait-for-finalize design decorative for the
99/// workloads it existed for. Quiescence short-circuits, so an idle module
100/// restarts immediately regardless of this value; the budget is spent only
101/// when a genuine in-flight request is worth finishing. Per-module override:
102/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
103/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
104/// where a stuck request will never settle).
105pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
106pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
107const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
108const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
109/// How long a drain waits after one busy-gauge probe completes before it asks
110/// again.
111///
112/// A drain re-probes on this short interval of its own rather than on the
113/// module's health cadence. The default cadence (30 s) equals the default
114/// drain ceiling, so a cadence-paced drain that found the module busy at its
115/// first probe never asked a second time: it waited out the whole ceiling and
116/// forced GOODBYE even after the gauges had read zero. The interval is counted
117/// from the moment the previous probe finished, and probes run one at a time,
118/// so a slow module is never asked more often than it can answer. Periodic
119/// health checks outside a drain keep using the configured cadence.
120const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
121/// How long a restart waits for the exited process's output readers.
122///
123/// The restart does not depend on either output reader finishing. A stderr
124/// reader still running at this bound is left running, and its later output goes
125/// into the exited process's own section of the stderr ring (see
126/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
127/// bound no longer decides whether a crash's last lines are kept: under load
128/// the reader may simply not have been scheduled yet, and cutting it there
129/// discarded exactly the lines that explained the crash.
130///
131/// What the bound still decides is when the tail starts reporting
132/// `Incomplete`: a pipe open past it usually means a descendant of the exited
133/// process still holds it, and the tail cannot claim to be whole until that
134/// pipe closes. That is also the only case in which the wait costs the restart
135/// anything, because a pipe with no other holder reaches EOF when the process
136/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
137/// `Captured` at EOF with nothing lost.
138///
139/// The stdout reader carries no ring, only the capture file. It also continues
140/// to EOF after this bound: child exit closes the writer but does not mean the
141/// reader has consumed all buffered bytes. Late output from either pipe can
142/// therefore reach the shared capture file after the successor starts.
143const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
144/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
145pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
146const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
147/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
148/// dropped because the subscriber stopped draining its frames. The detail's
149/// `first_undelivered_cursor` names the first event it did not receive; the
150/// client resubscribes from the last cursor it did receive.
151pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
152
153#[cfg(target_os = "macos")]
154struct PrivacyExec {
155    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
156    deadline: tokio::time::Instant,
157    expected: Option<subc_os::FileIdentity>,
158    trampoline: Option<subc_os::FileIdentity>,
159    script: bool,
160    module_id: String,
161    #[cfg(test)]
162    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
163}
164
165#[cfg(all(test, target_os = "macos"))]
166pub(crate) fn test_privacy_trampoline() -> PathBuf {
167    // The fixture bin lives in the profile folder the unit-test executable was
168    // built into. This honors custom CARGO_TARGET_DIR too.
169    let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
170    // Without the fixture every macOS spawn is refused, and the tests that
171    // spawn fail later as a module in state Failed, which names the wrong
172    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
173    assert!(
174        path.exists(),
175        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
176         --bins --features test-support` or `cargo test -p subc-daemon` first",
177        path.display()
178    );
179    path
180}
181
182#[cfg(target_os = "macos")]
183fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
184    use std::io::Read;
185    let mut probe = std::process::Command::new(path)
186        .args(["__disclaim-exec", "--probe"])
187        .stdin(Stdio::null())
188        .stdout(Stdio::piped())
189        .stderr(Stdio::piped())
190        .spawn()
191        .map_err(|error| {
192            format!(
193                "privacy trampoline probe failed for {}: {error}",
194                path.display()
195            )
196        })?;
197    let deadline = std::time::Instant::now() + Duration::from_secs(5);
198    let status = loop {
199        match probe.try_wait() {
200            Ok(Some(status)) => break status,
201            Ok(None) if std::time::Instant::now() < deadline => {
202                std::thread::sleep(Duration::from_millis(5))
203            }
204            result => {
205                let _ = probe.kill();
206                let _ = probe.wait();
207                return Err(format!(
208                    "privacy trampoline probe failed or timed out for {}: {result:?}",
209                    path.display()
210                ));
211            }
212        }
213    };
214    let mut answer = String::new();
215    if let Some(stdout) = probe.stdout.take() {
216        let _ = stdout.take(256).read_to_string(&mut answer);
217    }
218    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
219        return Ok(());
220    }
221    let mut diagnostic = String::new();
222    if let Some(stderr) = probe.stderr.take() {
223        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
224    }
225    let cause = diagnostic
226        .trim()
227        .strip_prefix("ck-subc: own privacy identity refused: ")
228        .unwrap_or("binary does not implement the privacy trampoline protocol");
229    Err(format!(
230        "{cause}: probe of {} exited {status}",
231        path.display()
232    ))
233}
234
235#[cfg(target_os = "macos")]
236fn privacy_command(
237    spec: &ModuleSpec,
238    roster: &ChildRoster,
239) -> Result<
240    (
241        Command,
242        Option<PrivacyExec>,
243        subc_os::privacy_identity::ExecAcknowledgement,
244    ),
245    SuperviseError,
246> {
247    let failure = |cause: String| {
248        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
249        SuperviseError::Spawn {
250            program: spec.program.clone(),
251            source: io::Error::other(cause),
252            cgroup_path: None,
253        }
254    };
255    let trampoline = roster.privacy_trampoline().map_err(failure)?;
256    // Resolve PATH with the same environment the Command will receive. For
257    // scripts retain the existing orphan-identity rule: the kernel chooses
258    // the interpreter, and its observed image is the one recorded. Do not
259    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
260    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
261        let path = spec
262            .env
263            .iter()
264            .find(|(key, _)| key == "PATH")
265            .map(|(_, value)| std::ffi::OsString::from(value))
266            .or_else(|| std::env::var_os("PATH"))
267            .unwrap_or_else(|| "/usr/bin:/bin".into());
268        std::env::split_paths(&path)
269            .map(|dir| dir.join(&spec.program))
270            .find(|path| path.is_file())
271            .unwrap_or_else(|| spec.program.clone())
272    } else {
273        spec.program.clone()
274    };
275    let expected = subc_os::file_identity(&program);
276    let trampoline_image = subc_os::file_identity(&trampoline);
277    let script = {
278        use std::io::Read;
279        let mut prefix = [0u8; 2];
280        std::fs::File::open(&program)
281            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
282    };
283    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
284        return Err(failure(
285            "privacy identity module executable is missing or is the trampoline itself".to_string(),
286        ));
287    }
288    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
289        .map_err(|error| failure(error.to_string()))?;
290    let reader =
291        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
292    let mut command = Command::new(&trampoline);
293    command
294        .arg("__disclaim-exec")
295        .arg(ack.fd().to_string())
296        .arg(&program);
297    Ok((
298        command,
299        Some(PrivacyExec {
300            reader,
301            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
302            expected,
303            trampoline: trampoline_image,
304            script,
305            module_id: spec.module_id.clone(),
306            #[cfg(test)]
307            confirmation_pause: spec
308                .env
309                .iter()
310                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
311                .map(|(_, address)| (address.clone(), None)),
312        }),
313        ack,
314    ))
315}
316
317struct SupervisedChild {
318    subtree_target: crate::subtree_resources::Target,
319    child: containment::ProcessChild,
320    exec: crate::activity::ExecOwner,
321    #[cfg(windows)]
322    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
323    #[cfg(target_os = "macos")]
324    privacy_exec: Option<PrivacyExec>,
325    /// Set once this launch's exec acknowledgement confirms the module image.
326    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
327    /// `subc_os::privacy_identity`), which then replaces itself with the
328    /// module. The supervisor owns and can kill that pid from spawn, but
329    /// status readers report it only after this latch is set, so nothing
330    /// reports the trampoline's image as the module's.
331    #[cfg(target_os = "macos")]
332    report_ready: Arc<OnceLock<()>>,
333    /// Supervisor failure detail retained for the terminal record, including
334    /// image refusals and first-HELLO timeouts after a successful spawn.
335    spawn_failure: Option<String>,
336    /// The protocol this process was launched with. A reload can store a new
337    /// launch spec with a different protocol, but that takes effect only at the
338    /// next spawn, so this process keeps being handled by the protocol it
339    /// actually speaks.
340    protocol: ModuleProtocol,
341    /// This container's cgroup name: a bounded module/slot label followed by a
342    /// unique suffix (when cgroup placement is on). Carried replacements reuse
343    /// the name; ordinary launches each get a fresh container. A
344    /// retired process in a slot may still be draining when a later one is
345    /// spawned into that slot, so the suffix keeps the later process out of
346    /// the retired one's cgroup, which is the domain a kill applies to.
347    #[cfg(target_os = "linux")]
348    module_id: String,
349    #[cfg(target_os = "linux")]
350    cgroup_placement: Option<subc_cgroup::Placement>,
351    /// The job for an ordinary, non-carried launch. For carried
352    /// launches the actor owns it and the child holds only a weak container link.
353    ///
354    /// Dropping this handle is what reaps a surviving tree when no supervisor
355    /// code runs — a daemon crash — because the job carries
356    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
357    ///
358    /// That limit is not crash-only, and the difference is worth knowing: a
359    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
360    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
361    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
362    /// module at once. Before this change they survived that, saw EOF on the
363    /// control socket, and ran their own teardown; Unix keeps that path
364    /// deliberately, so a module can seal a WAL or close a capture rather than
365    /// be killed mid-write. So this trades graceful teardown on every Windows
366    /// daemon stop for containment on a crash, which is the right way round
367    /// today: orphaned GPU workers are a reported, recurring problem, and the
368    /// modules that write most heavily do not run on Windows.
369    ///
370    /// The fix is a real Windows stop path — the daemon draining before it
371    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
372    /// reaches only what the drain left behind, which is what it should reach.
373    #[cfg(windows)]
374    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
375    stdout_pump: Option<JoinHandle<()>>,
376    stderr_pump: Option<StderrPump>,
377    stderr_ring: Arc<Mutex<StderrRing>>,
378    /// Conservative monotonic bound sampled before `command.spawn()`, not when
379    /// identity recording finishes or the spawn generation is published.
380    spawned_at: Instant,
381    /// Ordinary spawns must send their first HELLO within this budget. Once
382    /// accepted, a later disconnect must not restart the registration clock.
383    registration_deadline: Option<Instant>,
384    registration_watch: Option<RegistrationWatch>,
385    spawned_at_ms: u64,
386    spawned_from: PathBuf,
387    launch_spec_digest: LaunchSpecDigest,
388    spawned_file_identity: Option<SpawnedFileIdentity>,
389    process_start_time: Option<u64>,
390    process_identity: Option<ProcessIdentity>,
391    pid: u32,
392    /// This process's entry in the daemon's child roster, released when the
393    /// process is reaped or this handle is dropped.
394    roster_guard: Option<crate::child_roster::RosterGuard>,
395}
396
397impl SupervisedChild {
398    fn registered_since_spawn(&self) -> bool {
399        self.registration_watch
400            .as_ref()
401            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
402    }
403
404    fn id(&self) -> Option<u32> {
405        Some(self.pid)
406    }
407
408    fn process_identity(&self) -> Option<ProcessIdentity> {
409        self.process_identity
410    }
411
412    async fn wait(&mut self) -> io::Result<ExitStatus> {
413        #[cfg(target_os = "macos")]
414        self.confirm_privacy_exec().await;
415        // The roster entry is NOT released here. A daemon shutdown waits for the
416        // roster to empty and then exits the process, so releasing at the reap
417        // let it exit before the exit handler wrote this child's terminal record
418        // (the stderr drain and snapshot update sit in between), and the
419        // shutdown's own `daemon_shutdown` record was intermittently lost. The
420        // caller releases it after recording the exit (`release_roster`), and
421        // dropping the handle releases it too.
422        let result = self.child.wait().await;
423        #[cfg(target_os = "linux")]
424        if result.is_ok() && self.child.container().is_none() {
425            if let Some(placement) = self.cgroup_placement.as_ref() {
426                // Ordinary children still own a per-spawn cgroup: returning
427                // from wait must leave it empty and removed, as before carry.
428                // Keep cleanup ownership across cancellation until it finishes.
429                cleanup_reaped_cgroup(placement, &self.module_id).await;
430                self.cgroup_placement = None;
431            }
432        }
433        result
434    }
435
436    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
437        if self.child.container().is_some() {
438            if !carry {
439                self.child.terminate_container().await;
440            }
441        } else {
442            #[cfg(target_os = "linux")]
443            if let Some(placement) = self.cgroup_placement.as_ref() {
444                cleanup_reaped_cgroup(placement, &self.module_id).await;
445                self.cgroup_placement = None;
446            }
447        }
448        self.child.reap().await?;
449        self.release_roster();
450        Ok(())
451    }
452
453    #[cfg(target_os = "macos")]
454    async fn confirm_privacy_exec(&mut self) {
455        let Some(pending) = &mut self.privacy_exec else {
456            return;
457        };
458        let result = tokio::time::timeout_at(pending.deadline, async {
459            let mut record = Vec::new();
460            loop {
461                let mut ready = pending.reader.readable().await?;
462                let read = ready.try_io(|reader| {
463                    use std::io::Read;
464                    let mut reader = reader.get_ref();
465                    let mut buffer = [0u8; 256];
466                    reader.read(&mut buffer).map(|count| (count, buffer))
467                });
468                match read {
469                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
470                    Ok(Ok((count, buffer))) => {
471                        if record.len() + count > 1024 {
472                            return Err(io::Error::other(
473                                "privacy exec refusal record is too long",
474                            ));
475                        }
476                        record.extend_from_slice(&buffer[..count]);
477                    }
478                    Ok(Err(error)) => return Err(error),
479                    Err(_) => continue,
480                }
481            }
482        })
483        .await;
484        #[cfg(test)]
485        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
486            use tokio::io::{AsyncReadExt, AsyncWriteExt};
487            if stream.is_none() {
488                *stream = Some(
489                    tokio::net::TcpStream::connect(address.as_str())
490                        .await
491                        .unwrap(),
492                );
493                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
494                stream
495                    .as_mut()
496                    .unwrap()
497                    .write_all(&self.pid.to_ne_bytes())
498                    .await
499                    .unwrap();
500            }
501            let mut reply = [0];
502            stream
503                .as_mut()
504                .unwrap()
505                .read_exact(&mut reply)
506                .await
507                .unwrap();
508            pending.confirmation_pause = None;
509            if &reply == b"F" {
510                Ok(Err(io::Error::other("injected late exec refusal")))
511            } else {
512                assert_eq!(&reply, b"X");
513                result
514            }
515        } else {
516            result
517        };
518        // Keep the reader in self across await: select cancellation must not
519        // discard the handshake or reset its original five-second deadline.
520        let pending = self.privacy_exec.as_ref().expect("pending exec");
521        let cause = match result {
522            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
523            Ok(Err(error)) => Some(format!(
524                "privacy identity exec acknowledgement failed: {error}"
525            )),
526            Ok(Ok(record)) if !record.is_empty() => Some(
527                std::str::from_utf8(&record)
528                    .ok()
529                    .and_then(|record| {
530                        record
531                            .trim()
532                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
533                    })
534                    .filter(|cause| !cause.is_empty())
535                    .unwrap_or("invalid privacy exec refusal record")
536                    .to_string(),
537            ),
538            Ok(Ok(_)) => match self.child.try_wait() {
539                // Empty EOF is the exec acknowledgement. A real module may exit
540                // immediately, including with a reserved trampoline status; no
541                // image is admitted, and its ordinary exit contract stays intact.
542                Ok(Some(_status)) => None,
543                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
544                Ok(None) => {
545                    let image = observe_spawned_image(self.pid);
546                    if let Some(image) = image.filter(|image| {
547                        image.executable.is_some()
548                            && image.executable != pending.trampoline
549                            && (image.executable == pending.expected || pending.script)
550                    }) {
551                        if let Err(error) = self
552                            .roster_guard
553                            .as_ref()
554                            .map(|guard| guard.confirm_executable(image))
555                            .transpose()
556                        {
557                            Some(format!(
558                                "could not durably record confirmed module image: {error}"
559                            ))
560                        } else {
561                            let _ = self.report_ready.set(());
562                            info!(module_id = %pending.module_id, pid = self.pid,
563                                "module spawned with own privacy identity (responsibility disclaimed)");
564                            None
565                        }
566                    } else if image.is_none()
567                        || image.is_some_and(|image| image.executable.is_none())
568                    {
569                        // A process can exit between try_wait and the kernel
570                        // image read. Empty EOF already acknowledged exec, so
571                        // preserve that module's ordinary exit rather than
572                        // mislabel a disappearing image as trampoline refusal.
573                        // Keep pending in self across await so cancellation does
574                        // not discard validation or reset its original deadline.
575                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
576                            Ok(Ok(_status)) => None,
577                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
578                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
579                        }
580                    } else {
581                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
582                    }
583                }
584            },
585        };
586        let pending = self.privacy_exec.take().expect("pending exec");
587        // Exec confirmation describes launch success, not roster retirement.
588        // Keep even a reaped child recorded until its terminal history exists.
589        self.exec.0.finish(if cause.is_some() {
590            crate::activity::ExecState::Refused
591        } else {
592            crate::activity::ExecState::CommittedExec
593        });
594        if let Some(cause) = cause {
595            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
596            self.spawn_failure = Some(cause);
597            // No image is admitted on failure. Reach the entire fresh process
598            // group, including a module which spawned a helper before refusal.
599            if let Some(container) = self.child.container() {
600                container.hard_kill();
601            } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
602                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
603            }
604            let _ = self.child.start_kill();
605        }
606    }
607
608    /// Releases the leader's roster entry after recording and finalizing its
609    /// exit. A carried container retains its separate roster entry and, for a
610    /// Unix process group, its unreaped first leader that reserves the group ID.
611    fn release_roster(&mut self) {
612        self.roster_guard = None;
613    }
614
615    /// Kill the child and, where containment is available, its process tree.
616    ///
617    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
618    /// helper process leaked the helper — the Synapse embedding module's CUDA
619    /// worker holds the GPU allocation, so the leak cost VRAM until the next
620    /// restart of something else. Terminating the job reaches grandchildren that
621    /// a tree walk cannot, including one whose parent has already exited and
622    /// been reparented away.
623    ///
624    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
625    /// direct-child kill still decides the outcome, so containment can never
626    /// change whether a module is reported as stopped.
627    fn start_kill(&mut self) -> io::Result<()> {
628        if let Some(container) = self.child.container() {
629            container.hard_kill();
630            return self.kill_leader();
631        }
632        #[cfg(windows)]
633        if let Some(job) = &self.job {
634            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
635                debug!(
636                    error = %error,
637                    "job termination failed; the direct-child kill still owns the outcome"
638                );
639            }
640        }
641        #[cfg(target_os = "linux")]
642        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
643        self.child.start_kill()
644    }
645
646    fn kill_leader(&mut self) -> io::Result<()> {
647        self.child.start_kill()
648    }
649
650    async fn drain_stderr(&mut self, module_id: &str) {
651        if let Some(pump) = self.stdout_pump.take() {
652            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
653        }
654
655        let Some(pump) = self.stderr_pump.take() else {
656            return;
657        };
658        settle_stderr_pump(
659            module_id,
660            &self.stderr_ring,
661            pump,
662            STDERR_PUMP_DRAIN_TIMEOUT,
663        )
664        .await;
665    }
666}
667
668/// Bound the restart's wait, not the lifetime of the exited child's stdout
669/// reader. Slow pipe reads or capture writes must not turn the bound into lost
670/// output; dropping the handle lets the reader finish at EOF, like stderr.
671async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
672    match timeout(bound, &mut pump).await {
673        Ok(Ok(())) => {}
674        Ok(Err(error)) => {
675            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
676        }
677        Err(_) => {
678            drop(pump);
679            warn!(
680                module_id,
681                waited = ?bound,
682                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
683            );
684        }
685    }
686}
687
688/// The reader task for one process's stderr, with the ring generation its
689/// lines are attributed to.
690struct StderrPump {
691    task: JoinHandle<()>,
692    generation: u64,
693}
694
695/// Retire an exited process's stderr reader and wait up to `bound` for it to
696/// reach EOF. A reader still running at the bound is detached, not stopped: it
697/// keeps filling the exited process's section of the ring until its pipe
698/// closes, and the tail reads `Incomplete` until then. See
699/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
700async fn settle_stderr_pump(
701    module_id: &str,
702    ring: &Arc<Mutex<StderrRing>>,
703    pump: StderrPump,
704    bound: Duration,
705) {
706    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
707    let StderrPump {
708        mut task,
709        generation,
710    } = pump;
711    lock().retire_pump(generation);
712    match timeout(bound, &mut task).await {
713        Ok(Ok(())) => {}
714        Ok(Err(err)) => {
715            let mut ring = lock();
716            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
717            ring.finish_pump(generation);
718            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
719        }
720        Err(_) => {
721            // Dropping the handle detaches the task; it ends at EOF on its pipe.
722            drop(task);
723            lock().mark_pump_late(
724                generation,
725                format!(
726                    "stderr of the exited process had not reached EOF {bound:?} after it was \
727                     retired (a descendant may still hold the pipe open); lines it still \
728                     writes are kept in that process's section"
729                ),
730            );
731            warn!(
732                module_id,
733                waited = ?bound,
734                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
735            );
736        }
737    }
738}
739
740fn registration_release_events() -> &'static watch::Sender<u64> {
741    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
742    EVENTS.get_or_init(|| {
743        let (sender, _receiver) = watch::channel(0);
744        sender
745    })
746}
747
748pub(crate) fn notify_registration_release() {
749    let events = registration_release_events();
750    let next_generation = (*events.borrow()).wrapping_add(1);
751    events.send_replace(next_generation);
752}
753
754/// How to launch one singleton module process.
755#[derive(Debug, Clone, PartialEq, Eq)]
756pub struct ModuleSpec {
757    pub module_id: String,
758    pub program: PathBuf,
759    pub args: Vec<String>,
760    pub env: Vec<(String, String)>,
761    /// When true this is a reserved module: each spawn gets a fresh one-time launch
762    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
763    /// process can register this module_id (a security-boundary module like the
764    /// credential vault must not be impersonable while it is down/restarting).
765    pub reserved: bool,
766    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
767    /// Prefixes come from daemon config and must end in `:` before they reach the
768    /// supervisor; the owner module's current spawn nonce authorizes claims under
769    /// each prefix.
770    pub reserved_prefixes: Vec<String>,
771    /// The wire protocol this module speaks, as DECLARED in daemon config.
772    ///
773    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
774    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
775    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
776    /// and NO launch nonce, and a clean exit the daemon did not request is
777    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
778    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
779    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
780    /// because a process ignores an environment variable it does not read.
781    ///
782    /// The argument is the part that cannot be "harmless to a process that
783    /// ignores it": a stock binary exits on an unknown flag before it listens
784    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
785    /// first conformance run against this mode found it. The nonce is withheld
786    /// because a process that will never present it gains nothing from holding
787    /// it, and a secret in the environment of a process that does not need it is
788    /// a leak surface for no benefit.
789    pub protocol: ModuleProtocol,
790    /// Whether two processes of this module may run at once, which is what a
791    /// blue/green swap does for the length of its overlap. Declared in daemon
792    /// config because the daemon must be able to answer it while the module is
793    /// down, and so a module cannot talk itself into it after registering.
794    pub overlap: ModuleOverlap,
795    /// Keep module-started helpers in the same container across replacements.
796    /// This is a daemon-config policy, fixed for the lifetime of a container;
797    /// stopping the module or losing its supervisor still destroys the container.
798    pub carry_containment: bool,
799}
800
801/// Only the configured launch inputs enter this value; spawned connection paths,
802/// nonces and inherited environment never do. Its bytes remain daemon-private.
803#[derive(Clone, PartialEq, Eq)]
804pub(crate) struct LaunchSpecDigest([u8; 32]);
805
806impl fmt::Debug for LaunchSpecDigest {
807    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
808        f.write_str("LaunchSpecDigest([redacted])")
809    }
810}
811
812impl LaunchSpecDigest {
813    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
814        fn field(hash: &mut Sha256, bytes: &[u8]) {
815            hash.update((bytes.len() as u64).to_le_bytes());
816            hash.update(bytes);
817        }
818        let mut hash = Sha256::new();
819        if let Some(program) = spec.program.to_str() {
820            field(&mut hash, b"utf8-program");
821            field(&mut hash, program.as_bytes());
822        } else {
823            // The daemon config is UTF-8, but in-process callers may provide
824            // native paths: lossy conversion would collapse distinct programs.
825            #[cfg(unix)]
826            {
827                use std::os::unix::ffi::OsStrExt;
828                field(&mut hash, b"unix-program");
829                field(&mut hash, spec.program.as_os_str().as_bytes());
830            }
831            #[cfg(windows)]
832            {
833                use std::os::windows::ffi::OsStrExt;
834                field(&mut hash, b"windows-program");
835                let bytes = spec
836                    .program
837                    .as_os_str()
838                    .encode_wide()
839                    .flat_map(u16::to_le_bytes)
840                    .collect::<Vec<_>>();
841                field(&mut hash, &bytes);
842            }
843            #[cfg(not(any(unix, windows)))]
844            field(&mut hash, spec.program.to_string_lossy().as_bytes());
845        }
846        hash.update((spec.args.len() as u64).to_le_bytes());
847        for arg in &spec.args {
848            field(&mut hash, arg.as_bytes());
849        }
850        let mut env = spec.env.iter().collect::<Vec<_>>();
851        env.sort_unstable();
852        hash.update((env.len() as u64).to_le_bytes());
853        for (key, value) in env {
854            field(&mut hash, key.as_bytes());
855            field(&mut hash, value.as_bytes());
856        }
857        field(
858            &mut hash,
859            match spec.protocol {
860                ModuleProtocol::Subc => b"subc",
861                ModuleProtocol::None => b"none",
862            },
863        );
864        Self(hash.finalize().into())
865    }
866}
867
868/// Whether a module tolerates a second process of itself running alongside.
869///
870/// Most modules are single-writer on their store (a WAL, a capture log, a
871/// resident index behind a writer barrier), and two processes on one store
872/// corrupt it. So a swap, which overlaps the old and new process by design,
873/// is refused unless the module's config opts in with `overlap: "safe"`.
874#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
875pub enum ModuleOverlap {
876    /// Never run two processes of this module at once. The default.
877    #[default]
878    Exclusive,
879    /// The module has said a second process of itself is harmless for the
880    /// length of a swap.
881    ///
882    /// Declare it only if a second instance can run for a few seconds without
883    /// touching ANY single-writer store: every database, WAL, index, projector
884    /// and scheduled job the module owns. A lease on part of that state is not
885    /// enough. broca's session lease guards WAL appends while its run index, its
886    /// store projector and its archive fold timer (which unlinks live WAL files)
887    /// stay single-writer, so broca is exclusive despite holding a lease. The
888    /// refusal only fires after this has been decided, so the decision is the
889    /// check.
890    Safe,
891}
892
893impl ModuleOverlap {
894    pub fn as_str(self) -> &'static str {
895        match self {
896            Self::Exclusive => "exclusive",
897            Self::Safe => "safe",
898        }
899    }
900}
901
902/// Environment variable telling a spawned module which case it was started
903/// for, before it sends HELLO. Only a swap candidate carries it, as
904/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
905///
906/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
907/// longer because nobody waits on it, while a plain restart must flip ready
908/// quickly because callers see `module_warming` until it does. Absence means
909/// plain restart, the safe reading. The daemon trusts nothing about it; the
910/// candidate is proven by its launch nonce at HELLO.
911pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
912/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
913pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
914/// How long a swap waits for its candidate to register and declare itself
915/// ready when the operator does not say. A module warming as a swap candidate
916/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
917/// daemon allows that plus time to start the process and send HELLO.
918pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
919
920/// Bounded restart policy for crash exits.
921///
922/// `max_restarts` is the number of replacement processes allowed after the
923/// initial spawn WITHIN `window`. After that many crash restarts inside one
924/// window the module enters [`ModuleState::Failed`] and the supervisor stops
925/// the crash loop.
926///
927/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
928/// and that only survived because crashes were rare: a module that crashed
929/// three times across a week was disabled forever by crashes that had nothing
930/// to do with each other. That stopped being survivable once modules began
931/// exiting non-zero whenever the daemon's connection to them drops, because
932/// then every daemon-side connection drop spends a unit of the same budget and
933/// one flappy hour permanently stops a healthy module. Restarts older than
934/// `window` release their slot, so a module that crashed twice yesterday has a
935/// full budget today, while a genuine crash loop -- which is fast by
936/// definition -- still reaches the cap and stops.
937#[derive(Debug, Clone, Copy, PartialEq, Eq)]
938pub struct RestartPolicy {
939    pub max_restarts: u32,
940    /// Base delay before a crash replacement. The actual delay escalates with
941    /// the number of recent crash replacements and is capped by `max_backoff`.
942    pub backoff: Duration,
943    /// Maximum delay before a crash replacement.
944    pub max_backoff: Duration,
945    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
946    /// budget effectively infinite (nothing is ever in-window), which is why
947    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
948    pub window: Duration,
949}
950
951impl RestartPolicy {
952    /// A policy with the default crash window. Callers that care about the
953    /// window say so with [`Self::with_window`]; the ones that do not are
954    /// asking for the standard rate limit, not for no limit.
955    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
956        Self {
957            max_restarts,
958            backoff,
959            max_backoff: DEFAULT_MAX_BACKOFF,
960            window: DEFAULT_RESTART_WINDOW,
961        }
962    }
963
964    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
965        self.max_backoff = max_backoff;
966        self
967    }
968
969    pub fn with_window(mut self, window: Duration) -> Self {
970        self.window = window;
971        self
972    }
973
974    /// Calculate the capped exponential delay for the next crash replacement.
975    /// `restart_in_window` is zero for the first replacement after an operator
976    /// action (restart, reload, re-enable) cleared the crash ring, or after all
977    /// older crash replacements have aged out of the window.
978    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
979        if self.backoff.is_zero() || self.max_backoff.is_zero() {
980            return Duration::ZERO;
981        }
982
983        let mut delay = self.backoff;
984        for _ in 0..restart_in_window {
985            if delay >= self.max_backoff {
986                return self.max_backoff;
987            }
988            delay = delay
989                .checked_mul(10)
990                .unwrap_or(self.max_backoff)
991                .min(self.max_backoff);
992        }
993        delay.min(self.max_backoff)
994    }
995
996    /// The one sentence that explains a budget-exhausted stop, used for both the
997    /// log line and the terminal record so the two cannot drift. It names the
998    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
999    /// exactly what this budget is not.
1000    fn budget_exhausted_detail(&self) -> String {
1001        format!(
1002            "crash budget exhausted: max_restarts={} within window_secs={}",
1003            self.max_restarts,
1004            self.window.as_secs()
1005        )
1006    }
1007}
1008
1009impl Default for RestartPolicy {
1010    fn default() -> Self {
1011        Self {
1012            max_restarts: DEFAULT_MAX_RESTARTS,
1013            backoff: DEFAULT_BACKOFF,
1014            max_backoff: DEFAULT_MAX_BACKOFF,
1015            window: DEFAULT_RESTART_WINDOW,
1016        }
1017    }
1018}
1019
1020#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1021struct CrashRestartSchedule {
1022    restart_in_window: u32,
1023    delay: Duration,
1024}
1025
1026/// Whether the daemon itself will bring this module back after the exit being
1027/// handled: it is enabled AND its in-window crash restarts are below the cap.
1028///
1029/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1030/// the window are dropped here rather than by a timer, so the count is right
1031/// the moment somebody asks and no bookkeeping runs for idle modules.
1032fn daemon_will_restart(
1033    state: &mut SupervisorSnapshot,
1034    policy: &RestartPolicy,
1035    now: Instant,
1036) -> bool {
1037    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1038}
1039
1040const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1041const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1042const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1043const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1044
1045#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1046pub enum HealthAction {
1047    Report,
1048    Restart,
1049    Alert,
1050}
1051
1052impl fmt::Display for HealthAction {
1053    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1054        f.write_str(match self {
1055            Self::Report => "report",
1056            Self::Restart => "restart",
1057            Self::Alert => "alert",
1058        })
1059    }
1060}
1061
1062#[derive(Debug, Clone, PartialEq, Eq)]
1063pub struct HealthConfig {
1064    /// Optional loopback HTTP endpoint for a managed non-wire process.
1065    /// Changing it applies live on rescan; the process protocol changes only
1066    /// at its next spawn.
1067    pub http: Option<String>,
1068    pub cadence: Duration,
1069    pub deadline: Duration,
1070    pub failure_threshold: u32,
1071    pub on_degraded: HealthAction,
1072    pub on_failing: HealthAction,
1073    pub critical: bool,
1074}
1075
1076impl Default for HealthConfig {
1077    fn default() -> Self {
1078        Self {
1079            http: None,
1080            cadence: DEFAULT_HEALTH_CADENCE,
1081            deadline: DEFAULT_HEALTH_DEADLINE,
1082            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1083            on_degraded: HealthAction::Report,
1084            on_failing: HealthAction::Report,
1085            critical: false,
1086        }
1087    }
1088}
1089
1090/// The supervisor's view of one module's health, relayed to clients over
1091/// channel-0 and rendered by `ck health`.
1092///
1093/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1094/// stated here rather than only at the wire type a consumer reads. A reader can
1095/// look up what `None` means; only a writer can silently change it, and the
1096/// writer has no reason to go looking at a downstream contract before editing.
1097///
1098/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1099/// back to `None` on re-registration precisely so a respawned module does not
1100/// carry its predecessor's timestamp — so an old value and an absent one call for
1101/// opposite readings, and anything that defaulted this to a number would make a
1102/// never-probed module indistinguishable from one probed at the epoch.
1103///
1104/// `detail` and `metrics` are `None` when the module published none on this
1105/// probe, which does not mean it reported nothing wrong — it is also the shape
1106/// when the probe never reached it. `last_probe_ms` is what separates those.
1107#[derive(Debug, Clone, PartialEq)]
1108pub struct ModuleHealthStatus {
1109    pub status: SupervisorHealthStatus,
1110    pub last_probe_ms: Option<u64>,
1111    pub detail: Option<String>,
1112    pub metrics: Option<Value>,
1113    pub consecutive_failures: u32,
1114    /// Number of replies received after a recurring health probe's deadline.
1115    /// Unlike a timeout, every increment proves the module was alive.
1116    pub late_answer_count: u64,
1117    /// End-to-end latency of the newest late reply, measured from probe start.
1118    pub last_late_answer_latency_ms: Option<u64>,
1119    pub last_action: Option<String>,
1120    /// Set together with `last_action`; the pair moves as one, and both being
1121    /// absent means no escalation has ever been taken rather than that the last
1122    /// one succeeded.
1123    pub last_action_ms: Option<u64>,
1124}
1125
1126impl Default for ModuleHealthStatus {
1127    fn default() -> Self {
1128        Self {
1129            status: SupervisorHealthStatus::Unknown,
1130            last_probe_ms: None,
1131            detail: None,
1132            metrics: None,
1133            consecutive_failures: 0,
1134            late_answer_count: 0,
1135            last_late_answer_latency_ms: None,
1136            last_action: None,
1137            last_action_ms: None,
1138        }
1139    }
1140}
1141
1142/// Typed lifecycle state for a supervised module.
1143#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1144pub enum ModuleState {
1145    Starting,
1146    Running,
1147    Unresponsive,
1148    Restarting,
1149    Draining,
1150    Stopped,
1151    Failed,
1152    Disabled,
1153}
1154
1155impl fmt::Display for ModuleState {
1156    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1157        f.write_str(match self {
1158            Self::Starting => "starting",
1159            Self::Running => "running",
1160            Self::Unresponsive => "unresponsive",
1161            Self::Restarting => "restarting",
1162            Self::Draining => "draining",
1163            Self::Stopped => "stopped",
1164            Self::Failed => "failed",
1165            Self::Disabled => "disabled",
1166        })
1167    }
1168}
1169
1170/// Supervisor classification of a child-process exit.
1171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1172pub enum ExitKind {
1173    Clean,
1174    Crash,
1175    DeliberateSeverance,
1176}
1177
1178impl From<ExitKind> for TerminalExitKind {
1179    fn from(kind: ExitKind) -> Self {
1180        match kind {
1181            ExitKind::Clean => Self::Clean,
1182            ExitKind::Crash => Self::Crash,
1183            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1184        }
1185    }
1186}
1187
1188/// Exact process identity retained when a supervised module registers its
1189/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1190#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1191pub(crate) struct ProcessIdentity {
1192    pub(crate) pid: u32,
1193    pub(crate) start_time: u64,
1194}
1195
1196/// Last observed child exit, if any.
1197#[derive(Debug, Clone, PartialEq, Eq)]
1198pub struct ExitReport {
1199    pub kind: ExitKind,
1200    pub code: Option<i32>,
1201    pub signal: Option<i32>,
1202    pub at_ms: u64,
1203}
1204
1205/// Point-in-time module status answerable by subc without forwarding to the
1206/// module process.
1207#[derive(Debug, Clone, PartialEq)]
1208pub struct ModuleStatus {
1209    pub module_id: String,
1210    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1211    pub waiting_for_previous_pid: Option<u32>,
1212    /// Why the previous instance still prevents a new start.
1213    pub waiting_for_previous_detail: Option<String>,
1214    /// The daemon's Windows per-spawn record: `pipe` after the process it
1215    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1216    /// List/status reports use this record, not the source the module sends
1217    /// in its registration message's manifest.provenance.launch_nonce_source.
1218    pub launch_nonce_source: Option<String>,
1219    pub state: ModuleState,
1220    pub enabled: bool,
1221    pub process_alive: bool,
1222    pub registration_active: bool,
1223    /// The module's declared wire protocol, carried beside `live` because it is
1224    /// what makes `live` readable: the two fields answer one question together.
1225    /// While a process is alive this is its launch declaration, not a later
1226    /// pending-reload edit. When down it is the configured next launch protocol.
1227    pub protocol: ModuleProtocol,
1228    /// Whether the module is serving, under the strongest definition the daemon
1229    /// can assert for its protocol.
1230    ///
1231    /// A subc module must also be REGISTERED: its process being alive says
1232    /// nothing about whether it can take a request. A `protocol: "none"` module
1233    /// never registers, so that term is dropped and this falls back to "enabled,
1234    /// running, and the process the daemon launched is alive" -- which is all
1235    /// the daemon observes about a process that speaks no subc wire. It stays a
1236    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1237    /// rather than printing it bare.
1238    pub live: bool,
1239    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1240    /// restarts have already released their slot, so this count can go down
1241    /// without anybody touching the module.
1242    pub restart_count: u32,
1243    /// Replacement processes spawned over this module's entire supervisor lifetime;
1244    /// unlike `restart_count`, this value is never reset by an operator action
1245    /// and never falls out of a window.
1246    pub lifetime_restarts: u32,
1247    pub spawn_generation: u64,
1248    /// The budget `restart_count` is spent against. Carried alongside the count
1249    /// because the count alone does not say how close the module is to being
1250    /// disabled, and reporting one without the other is what makes an
1251    /// about-to-be-retired module look ordinary.
1252    pub max_restarts: u32,
1253    /// The span `restart_count` is counted over. Carried with the pair above for
1254    /// the same reason they are carried together: "2 of 3" means one thing for a
1255    /// ten-minute window and something else entirely for a lifetime.
1256    pub restart_window: Duration,
1257    /// Effective drain and restart timing policy used by this running module.
1258    /// These values are carried together with the restart budget so status
1259    /// readers can compare configured intent with what the supervisor applied.
1260    pub drain_timeout: Duration,
1261    pub restart_backoff: Duration,
1262    pub restart_max_backoff: Duration,
1263    /// The module's process. On macOS this stays absent while the `ck-subc`
1264    /// launch trampoline is still running in that pid, and appears once the
1265    /// exec acknowledgement confirms the module image has replaced it. Launch
1266    /// time and the supervisor's own hold on the process are unaffected.
1267    pub pid: Option<u32>,
1268    pub spawned_at_ms: Option<u64>,
1269    pub spawned_from: Option<PathBuf>,
1270    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1271    pub process_start_time: Option<u64>,
1272    pub last_exit: Option<ExitReport>,
1273    pub health: ModuleHealthStatus,
1274}
1275
1276#[cfg(test)]
1277#[derive(Debug, Clone, Copy, PartialEq)]
1278struct ActorSelectCheckpoint {
1279    generation: u64,
1280    turn: u64,
1281    registered_connection: Option<ConnectionId>,
1282    next_probe_at: Option<Instant>,
1283    wake_after: Option<Duration>,
1284}
1285
1286#[derive(Debug, Clone, PartialEq)]
1287struct SupervisorSnapshot {
1288    subtree_target: crate::subtree_resources::Target,
1289    failed_retry: FailedRetry,
1290    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1291    #[cfg(windows)]
1292    pipe_consumed: Option<Arc<OnceLock<()>>>,
1293    /// Counts running-child loop turns, not executor polls of a parked wait.
1294    #[cfg(test)]
1295    actor_turns: u64,
1296    /// Running select after exec confirmation and with no unconsumed registry
1297    /// event. Recording a spawn or starting a loop turn is not this barrier.
1298    #[cfg(test)]
1299    actor_select: Option<ActorSelectCheckpoint>,
1300    state: ModuleState,
1301    enabled: bool,
1302    process_alive: bool,
1303    spawned_protocol: Option<ModuleProtocol>,
1304    spawn_failure: Option<String>,
1305    waiting_for_previous_pid: Option<u32>,
1306    waiting_for_previous_detail: Option<String>,
1307    /// When each crash restart was spent, oldest first. This IS the crash
1308    /// budget: its in-window length is the count an operator sees and the count
1309    /// the restart decision is made against, so there is no second counter that
1310    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1311    /// operator actions that used to zero the old lifetime counter.
1312    crash_restarts: VecDeque<Instant>,
1313    lifetime_restarts: u32,
1314    /// Successful child spawns in this daemon incarnation.
1315    ///
1316    /// `lifetime_restarts` was considered and rejected: it starts at zero
1317    /// (line 640), successful initial/operator spawns in `set_running` do not
1318    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1319    /// increments before a successful replacement exists (lines 604, 3846,
1320    /// and 3921), so a failed spawn can consume it. This counter moves only
1321    /// when a live PID is accepted below.
1322    spawn_generation: u64,
1323    pid: Option<u32>,
1324    #[cfg(target_os = "macos")]
1325    report_ready: Option<Arc<OnceLock<()>>>,
1326    /// Last reaped child, retained after current process facts are cleared.
1327    reaped_pid: Option<u32>,
1328    /// Whether a replacement has already been scheduled for the current process.
1329    respawn_pending: bool,
1330    /// A second restart is waiting for the replacement already scheduled.
1331    coalesced_restart_pending: bool,
1332    spawned_at_ms: Option<u64>,
1333    spawned_from: Option<PathBuf>,
1334    launch_spec_digest: Option<LaunchSpecDigest>,
1335    spawned_file_identity: Option<SpawnedFileIdentity>,
1336    process_start_time: Option<u64>,
1337    deliberate_severance: Option<ProcessIdentity>,
1338    last_exit: Option<ExitReport>,
1339    /// Diagnostic attached to the next drain's terminal record, if any.
1340    drain_disposition_detail: Option<String>,
1341    health: ModuleHealthStatus,
1342    last_probe_evidence: Option<HealthProbeEvidence>,
1343    /// Whether the current process was started as a swap candidate and so
1344    /// lives in the module's alternate cgroup. The next swap's candidate takes
1345    /// the other one, so the two processes of a swap never share a cgroup. A
1346    /// plain spawn always uses the primary cgroup.
1347    in_alternate_slot: bool,
1348    /// Whether the current `Draining` state ends in a replacement process
1349    /// (restart, reload, health restart) rather than a stop. Only meaningful
1350    /// while `state` is `Draining`; every entry into that state rewrites it.
1351    /// It is what lets route.open answer the retryable `module_reloading` to a
1352    /// consumer that reaches a still-registered process mid-restart, instead of
1353    /// the `supervisor_not_live` a stop or disable deserves.
1354    draining_to_replace: bool,
1355    /// Whether a configuration update has been applied since the current
1356    /// process was spawned, so that process runs an older spec than the one
1357    /// the supervisor now holds. A queued restart is only coalesced into a
1358    /// fresher process when this is false: a restart requested to pick up a
1359    /// new configuration must not be satisfied by a process that predates it.
1360    configuration_updated_since_spawn: bool,
1361}
1362
1363impl SupervisorSnapshot {
1364    /// The pid that status, provenance and resource readings may report. While
1365    /// the launch trampoline still runs in the pid, reading its executable or
1366    /// resource use would describe `ck-subc`, not the module, so none is
1367    /// reported until the exec acknowledgement confirms the module image.
1368    fn reported_pid(&self) -> Option<u32> {
1369        #[cfg(target_os = "macos")]
1370        if self
1371            .report_ready
1372            .as_ref()
1373            .is_some_and(|ready| ready.get().is_none())
1374        {
1375            return None;
1376        }
1377        self.pid
1378    }
1379
1380    fn starting() -> Self {
1381        Self::new(ModuleState::Starting, true)
1382    }
1383
1384    fn disabled() -> Self {
1385        Self::new(ModuleState::Disabled, false)
1386    }
1387
1388    /// Crash restarts still inside `window`, having dropped the ones that are
1389    /// not. Pruning on read is what makes the budget a rate: an instant older
1390    /// than the window stops holding a slot the moment anybody counts.
1391    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1392        while let Some(oldest) = self.crash_restarts.front() {
1393            if now.duration_since(*oldest) > window {
1394                self.crash_restarts.pop_front();
1395            } else {
1396                break;
1397            }
1398        }
1399        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1400    }
1401
1402    /// Spend one unit of the crash budget and record the restart in the ledger.
1403    ///
1404    /// The ring is bounded by the cap because more than `max_restarts` in-window
1405    /// instants can never be reached (the caller refuses the restart first), so
1406    /// anything beyond that is an unbounded queue waiting to happen.
1407    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1408        self.crash_restarts.push_back(now);
1409        while self.crash_restarts.len() > policy.max_restarts as usize {
1410            self.crash_restarts.pop_front();
1411        }
1412        self.lifetime_restarts += 1;
1413    }
1414
1415    /// Reserve one crash-restart slot and calculate the delay before respawning.
1416    /// The count is captured before recording this restart, so the first retry
1417    /// uses the base delay and each later in-window retry escalates once.
1418    fn next_crash_restart(
1419        &mut self,
1420        policy: &RestartPolicy,
1421        now: Instant,
1422    ) -> Option<CrashRestartSchedule> {
1423        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1424        if restart_in_window >= policy.max_restarts {
1425            return None;
1426        }
1427        self.record_crash_restart(policy, now);
1428        Some(CrashRestartSchedule {
1429            restart_in_window,
1430            delay: policy.delay_for_restart(restart_in_window),
1431        })
1432    }
1433
1434    /// Give the module its full budget back, as an operator restart, reload, or
1435    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1436    /// ledger of what actually happened, and an operator action does not unmake
1437    /// the crashes.
1438    fn clear_crash_restarts(&mut self) {
1439        self.crash_restarts.clear();
1440    }
1441
1442    fn new(state: ModuleState, enabled: bool) -> Self {
1443        Self {
1444            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1445            failed_retry: FailedRetry::default(),
1446            exec_outcome: None,
1447            #[cfg(test)]
1448            actor_turns: 0,
1449            #[cfg(test)]
1450            actor_select: None,
1451            state,
1452            enabled,
1453            process_alive: false,
1454            #[cfg(windows)]
1455            pipe_consumed: None,
1456            spawned_protocol: None,
1457            spawn_failure: None,
1458            waiting_for_previous_pid: None,
1459            waiting_for_previous_detail: None,
1460            crash_restarts: VecDeque::new(),
1461            lifetime_restarts: 0,
1462            spawn_generation: 0,
1463            pid: None,
1464            #[cfg(target_os = "macos")]
1465            report_ready: None,
1466            reaped_pid: None,
1467            respawn_pending: false,
1468            coalesced_restart_pending: false,
1469            spawned_at_ms: None,
1470            spawned_from: None,
1471            launch_spec_digest: None,
1472            spawned_file_identity: None,
1473            process_start_time: None,
1474            deliberate_severance: None,
1475            last_exit: None,
1476            drain_disposition_detail: None,
1477            health: ModuleHealthStatus::default(),
1478            last_probe_evidence: None,
1479            in_alternate_slot: false,
1480            draining_to_replace: false,
1481            configuration_updated_since_spawn: false,
1482        }
1483    }
1484}
1485
1486type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1487
1488type SpawnSubscriberKey = (ConnectionId, u64);
1489
1490#[derive(Debug)]
1491struct SpawnSubscriber {
1492    version: u8,
1493    frames: mpsc::Sender<Frame>,
1494    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1495    /// from which event. The full frame channel cannot carry that news, so it
1496    /// travels beside it; see `SpawnEventFeed::subscribe`.
1497    lagged: Option<oneshot::Sender<SpawnCursor>>,
1498}
1499
1500#[derive(Debug)]
1501struct SpawnEventState {
1502    daemon_incarnation: String,
1503    seq: u64,
1504    capacity: usize,
1505    live: HashMap<String, LiveSpawn>,
1506    generations: HashMap<String, u64>,
1507    events: VecDeque<SpawnEvent>,
1508    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1509}
1510
1511impl Default for SpawnEventState {
1512    fn default() -> Self {
1513        Self {
1514            daemon_incarnation: "unconfigured".to_string(),
1515            seq: 0,
1516            capacity: SPAWN_EVENT_RING_CAPACITY,
1517            live: HashMap::new(),
1518            generations: HashMap::new(),
1519            events: VecDeque::new(),
1520            subscribers: HashMap::new(),
1521        }
1522    }
1523}
1524
1525#[derive(Debug, Clone, Default)]
1526struct SpawnEventFeed(
1527    Arc<Mutex<SpawnEventState>>,
1528    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1529);
1530
1531#[derive(Debug, Clone, PartialEq, Eq)]
1532pub(crate) enum SpawnSubscribeRefusal {
1533    ForeignIncarnation { current: String },
1534    TooOld { oldest: SpawnCursor },
1535    Frame(String),
1536}
1537
1538impl SpawnEventFeed {
1539    fn configure_incarnation(&self, daemon_incarnation: String) {
1540        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1541        state.daemon_incarnation = daemon_incarnation;
1542        state.seq = 0;
1543        state.live.clear();
1544        state.generations.clear();
1545        state.events.clear();
1546        state.subscribers.clear();
1547    }
1548
1549    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1550        SpawnCursor {
1551            daemon_incarnation: state.daemon_incarnation.clone(),
1552            seq: state.seq,
1553        }
1554    }
1555
1556    fn snapshot(&self) -> SpawnSnapshot {
1557        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1558        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1559        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1560        SpawnSnapshot {
1561            cursor: Self::cursor(&state),
1562            ring_bound: state.capacity as u64,
1563            live,
1564        }
1565    }
1566
1567    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1568        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1569        let generation = state
1570            .generations
1571            .get(module_id)
1572            .copied()
1573            .unwrap_or(0)
1574            .checked_add(1)
1575            .expect("spawn generation exhausted");
1576        state.generations.insert(module_id.to_string(), generation);
1577        let live = LiveSpawn {
1578            module_id: module_id.to_string(),
1579            spawn_generation: generation,
1580            pid,
1581            spawned_at_ms,
1582        };
1583        state.live.insert(module_id.to_string(), live);
1584        Self::emit_locked(
1585            &mut state,
1586            SpawnEventKind::Spawned,
1587            module_id.to_string(),
1588            generation,
1589            pid,
1590            None,
1591            None,
1592        );
1593        generation
1594    }
1595
1596    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1597        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1598        let Some(live) = state.live.remove(module_id) else {
1599            warn!(
1600                module_id,
1601                "terminal record had no live spawn event identity"
1602            );
1603            return;
1604        };
1605        Self::emit_locked(
1606            &mut state,
1607            SpawnEventKind::Exited,
1608            module_id.to_string(),
1609            live.spawn_generation,
1610            live.pid,
1611            exit_code,
1612            exit_signal,
1613        );
1614    }
1615
1616    /// Report the exit of a process that a swap has already replaced.
1617    ///
1618    /// `emit_exited` removes the module's live entry, which after a swap's
1619    /// cutover describes the promoted candidate, not the old process now
1620    /// exiting. This emits the old generation's exit and leaves the live entry
1621    /// alone unless it still names that generation.
1622    fn emit_superseded_exited(
1623        &self,
1624        module_id: &str,
1625        spawn_generation: u64,
1626        pid: u32,
1627        exit_code: Option<i32>,
1628        exit_signal: Option<i32>,
1629    ) {
1630        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1631        if state
1632            .live
1633            .get(module_id)
1634            .is_some_and(|live| live.spawn_generation == spawn_generation)
1635        {
1636            state.live.remove(module_id);
1637        }
1638        Self::emit_locked(
1639            &mut state,
1640            SpawnEventKind::Exited,
1641            module_id.to_string(),
1642            spawn_generation,
1643            pid,
1644            exit_code,
1645            exit_signal,
1646        );
1647    }
1648
1649    #[allow(clippy::too_many_arguments)]
1650    fn emit_locked(
1651        state: &mut SpawnEventState,
1652        kind: SpawnEventKind,
1653        module_id: String,
1654        spawn_generation: u64,
1655        pid: u32,
1656        exit_code: Option<i32>,
1657        exit_signal: Option<i32>,
1658    ) {
1659        state.seq = state
1660            .seq
1661            .checked_add(1)
1662            .expect("spawn event sequence exhausted");
1663        let event = SpawnEvent {
1664            cursor: Self::cursor(state),
1665            kind,
1666            module_id,
1667            spawn_generation,
1668            pid,
1669            exit_code,
1670            exit_signal,
1671        };
1672        state.events.push_back(event.clone());
1673        while state.events.len() > state.capacity {
1674            state.events.pop_front();
1675        }
1676        let body = match serde_json::to_vec(&event) {
1677            Ok(body) => body,
1678            Err(error) => {
1679                error!(%error, "failed to serialize supervisor spawn event");
1680                return;
1681            }
1682        };
1683        state.subscribers.retain(|(connection_id, corr), subscriber| {
1684            let frame = Frame::build_with_version(
1685                subscriber.version,
1686                FrameType::StreamData,
1687                control_flags(),
1688                0,
1689                0,
1690                *corr,
1691                body.clone(),
1692            );
1693            match frame {
1694                Ok(frame) => {
1695                    if subscriber.frames.try_send(frame).is_ok() {
1696                        true
1697                    } else {
1698                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1699                        if let Some(lagged) = subscriber.lagged.take() {
1700                            let _ = lagged.send(event.cursor.clone());
1701                        }
1702                        false
1703                    }
1704                }
1705                Err(error) => {
1706                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1707                    false
1708                }
1709            }
1710        });
1711    }
1712
1713    fn subscribe(
1714        &self,
1715        connection_id: ConnectionId,
1716        corr: u64,
1717        version: u8,
1718        since: Option<SpawnCursor>,
1719        sink: FrameSink,
1720    ) -> Result<(), SpawnSubscribeRefusal> {
1721        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1722        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1723        {
1724            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1725            let replay = if let Some(since) = since {
1726                if since.daemon_incarnation != state.daemon_incarnation {
1727                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1728                        current: state.daemon_incarnation.clone(),
1729                    });
1730                }
1731                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1732                    if since.seq < oldest.seq.saturating_sub(1) {
1733                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1734                    }
1735                }
1736                state
1737                    .events
1738                    .iter()
1739                    .filter(|event| event.cursor.seq > since.seq)
1740                    .cloned()
1741                    .collect::<Vec<_>>()
1742            } else {
1743                Vec::new()
1744            };
1745            for event in replay {
1746                let body = serde_json::to_vec(&event)
1747                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1748                let frame = Frame::build_with_version(
1749                    version,
1750                    FrameType::StreamData,
1751                    control_flags(),
1752                    0,
1753                    0,
1754                    corr,
1755                    body,
1756                )
1757                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1758                frames
1759                    .try_send(frame)
1760                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1761            }
1762            state.subscribers.insert(
1763                (connection_id, corr),
1764                SpawnSubscriber {
1765                    version,
1766                    frames,
1767                    lagged: Some(lagged),
1768                },
1769            );
1770        }
1771        // The lagged terminal is sent here, by the forwarder, rather than by
1772        // the emitter: at the moment of the drop the subscriber's own channel
1773        // is full, and writing to the connection sink directly from the emitter
1774        // would put the Error AHEAD of the events still queued in that channel
1775        // (and the emitter holds the feed lock, so it cannot await the sink).
1776        // Dropping the subscriber drops the only sender, so `recv` drains every
1777        // queued event and then returns `None`; only then is the Error sent, so
1778        // the client sees each event it can keep, then the reason it was cut.
1779        // Cancel and connection removal drop the oneshot unsent, so they end
1780        // the stream with no Error.
1781        tokio::spawn(async move {
1782            while let Some(frame) = receiver.recv().await {
1783                if sink.send(frame).await.is_err() {
1784                    return;
1785                }
1786            }
1787            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1788                return;
1789            };
1790            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1791                Ok(frame) => {
1792                    let _ = sink.send(frame).await;
1793                }
1794                Err(error) => {
1795                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1796                }
1797            }
1798        });
1799        Ok(())
1800    }
1801
1802    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1803        let Some(subscriber) = self
1804            .0
1805            .lock()
1806            .unwrap_or_else(|p| p.into_inner())
1807            .subscribers
1808            .remove(&(connection_id, corr))
1809        else {
1810            return false;
1811        };
1812        if let Ok(frame) = Frame::build_with_version(
1813            subscriber.version,
1814            FrameType::StreamEnd,
1815            control_flags(),
1816            0,
1817            0,
1818            corr,
1819            Vec::new(),
1820        ) {
1821            tokio::spawn(async move {
1822                let _ = subscriber.frames.send(frame).await;
1823            });
1824        }
1825        true
1826    }
1827
1828    fn remove_connection(&self, connection_id: ConnectionId) {
1829        self.0
1830            .lock()
1831            .unwrap_or_else(|p| p.into_inner())
1832            .subscribers
1833            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1834    }
1835
1836    #[cfg(any(test, feature = "test-support"))]
1837    fn set_capacity(&self, capacity: usize) {
1838        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1839    }
1840
1841    #[cfg(any(test, feature = "test-support"))]
1842    fn subscriber_count(&self) -> usize {
1843        self.0
1844            .lock()
1845            .unwrap_or_else(|p| p.into_inner())
1846            .subscribers
1847            .len()
1848    }
1849}
1850
1851/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1852/// The terminal Error a lagged spawn subscriber receives after its queued events.
1853fn spawn_subscriber_lagged_frame(
1854    version: u8,
1855    corr: u64,
1856    first_undelivered: SpawnCursor,
1857) -> Result<Frame, String> {
1858    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1859    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1860        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1861        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1862            .to_string(),
1863        detail: Some(serde_json::json!({
1864            "first_undelivered_cursor": first_undelivered
1865        })),
1866    })
1867    .map_err(|error| error.to_string())?;
1868    Frame::build_with_version(
1869        version,
1870        FrameType::Error,
1871        crate::router::daemon_error_flags(),
1872        0,
1873        0,
1874        corr,
1875        body,
1876    )
1877    .map_err(|error| error.to_string())
1878}
1879
1880pub trait ModuleProcessLiveness: Send + Sync {
1881    fn process_live(&self, module_id: &str) -> Option<bool>;
1882
1883    /// Whether the supervisor is replacing this module's process right now: an
1884    /// operator restart or reload, a health restart, or a crash respawn whose
1885    /// backoff is running. A module in that state is not live, but a consumer
1886    /// refused now should retry shortly rather than treat the target as gone.
1887    /// Stopped, failed, and disabled modules are not replacing.
1888    fn process_replacing(&self, _module_id: &str) -> bool {
1889        false
1890    }
1891}
1892
1893/// Shared process-liveness registry keyed by supervised `module_id`.
1894#[derive(Debug, Clone, Default)]
1895pub struct SupervisorProcessLiveness {
1896    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1897}
1898
1899impl SupervisorProcessLiveness {
1900    pub fn new() -> Self {
1901        Self::default()
1902    }
1903
1904    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1905        let mut snapshots = self
1906            .snapshots
1907            .lock()
1908            .unwrap_or_else(|poisoned| poisoned.into_inner());
1909        snapshots.insert(module_id, snapshot);
1910    }
1911
1912    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1913        let mut snapshots = self
1914            .snapshots
1915            .lock()
1916            .unwrap_or_else(|poisoned| poisoned.into_inner());
1917        let is_current = snapshots
1918            .get(module_id)
1919            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1920            .unwrap_or(false);
1921        if is_current {
1922            snapshots.remove(module_id);
1923        }
1924    }
1925}
1926
1927impl ModuleProcessLiveness for SupervisorProcessLiveness {
1928    fn process_live(&self, module_id: &str) -> Option<bool> {
1929        let snapshot = {
1930            let snapshots = self
1931                .snapshots
1932                .lock()
1933                .unwrap_or_else(|poisoned| poisoned.into_inner());
1934            snapshots.get(module_id).cloned()
1935        }?;
1936        let snapshot = snapshot
1937            .lock()
1938            .unwrap_or_else(|poisoned| poisoned.into_inner());
1939        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1940    }
1941
1942    fn process_replacing(&self, module_id: &str) -> bool {
1943        let Some(snapshot) = self
1944            .snapshots
1945            .lock()
1946            .unwrap_or_else(|poisoned| poisoned.into_inner())
1947            .get(module_id)
1948            .cloned()
1949        else {
1950            return false;
1951        };
1952        let snapshot = snapshot
1953            .lock()
1954            .unwrap_or_else(|poisoned| poisoned.into_inner());
1955        snapshot.enabled
1956            && match snapshot.state {
1957                ModuleState::Restarting => true,
1958                ModuleState::Draining => snapshot.draining_to_replace,
1959                ModuleState::Starting
1960                | ModuleState::Running
1961                | ModuleState::Unresponsive
1962                | ModuleState::Stopped
1963                | ModuleState::Failed
1964                | ModuleState::Disabled => false,
1965            }
1966    }
1967}
1968
1969#[cfg(test)]
1970#[derive(Debug, Default)]
1971struct ReloadExitRecordGate {
1972    reached: tokio::sync::Notify,
1973    resume: tokio::sync::Notify,
1974}
1975
1976#[derive(Debug, Clone, Copy)]
1977enum RespawnKind {
1978    Spawn,
1979    Reload,
1980}
1981
1982#[derive(Debug, Clone, Copy)]
1983struct PendingRespawn {
1984    deadline: Instant,
1985    kind: RespawnKind,
1986    daemon_initiated: bool,
1987}
1988
1989type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1990
1991#[derive(Debug, Clone)]
1992struct SupervisorRuntimeConfig {
1993    registry: Arc<Registry>,
1994    /// Retained across an operation's deferred backoff, until its child starts.
1995    recovery_is_daemon: bool,
1996    start_requester: Option<Principal>,
1997    pending_activity_restart: bool,
1998    provenance_probe: ExecutableIdentityProbe,
1999    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2000    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2001    /// A reload acknowledges completion only after its replacement registers.
2002    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2003    restart_policy: RestartPolicy,
2004    register_timeout: Duration,
2005    /// This module's RESOLVED drain budget: per-module config when present,
2006    /// else `default_drain_timeout`.
2007    drain_timeout: Duration,
2008    /// Shared with the status handle so the attested value changes atomically
2009    /// when a rescan updates the running drain policy.
2010    effective_drain_timeout: Arc<Mutex<Duration>>,
2011    /// The supervisor-wide fallback, kept so a configuration update that
2012    /// REMOVES the per-module override can re-resolve to it.
2013    default_drain_timeout: Duration,
2014    health: HealthConfig,
2015    host_health: HostHealth,
2016    connection_file_path: Option<PathBuf>,
2017    capture_logs_dir: Option<PathBuf>,
2018    forwarding: Option<Arc<ForwardingTable>>,
2019    /// The shared handle, so every spawn path (initial, restart, reload) records the
2020    /// reserved-module launch nonce the HELLO verifier checks against.
2021    supervisor_handle: Option<SupervisorHandle>,
2022    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2023    /// status queries.
2024    ///
2025    /// One ring per module, held across every respawn. The lines explaining an exit
2026    /// are written BEFORE that exit, so a ring recreated per process would be empty
2027    /// exactly when it is asked for.
2028    stderr_ring: Arc<Mutex<StderrRing>>,
2029    terminal_ring: Arc<Mutex<TerminalRing>>,
2030    spawn_events: SpawnEventFeed,
2031    child_roster: ChildRoster,
2032    #[cfg(target_os = "linux")]
2033    cgroup_placement: Option<subc_cgroup::Placement>,
2034    #[cfg(test)]
2035    test_seed_stale_facts_before_enable_spawn: bool,
2036    #[cfg(test)]
2037    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2038}
2039
2040impl SupervisorRuntimeConfig {
2041    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2042        FailureContext {
2043            daemon_initiated,
2044            shutdown: self.child_roster.shutdown_flag(),
2045        }
2046    }
2047}
2048
2049#[derive(Debug, Clone, PartialEq, Eq)]
2050struct SupervisedConfiguration {
2051    spec: ModuleSpec,
2052    health: HealthConfig,
2053}
2054
2055/// Shared daemon lookup table for supervised module handles.
2056///
2057/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2058/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2059/// launch nonces recorded at spawn are checked by the same daemon instance.
2060#[derive(Debug, Clone, Default)]
2061pub struct SupervisorHandle {
2062    host_health: HostHealth,
2063    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2064    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2065    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2066    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2067    /// Module ids the supervisor has taken on. An id is added BEFORE the
2068    /// module's first process is spawned and removed only when the module
2069    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2070    /// the keys of `modules`.
2071    ///
2072    /// `modules` cannot answer "is this module configured?" on its own: a
2073    /// [`SupervisedModule`] only exists once its process has been spawned, and
2074    /// a fast child can connect, register, sync its scopes and ask about them
2075    /// before the supervisor has inserted it. Answering "not configured" in that
2076    /// gap makes scope admission refuse with the terminal "will never sync"
2077    /// instead of the retryable "has not synced yet".
2078    configured_ids: Arc<Mutex<HashSet<String>>>,
2079    spawn_events: SpawnEventFeed,
2080    /// The current expected launch nonce for each reserved module_id. Set when the
2081    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2082    /// non-reserved module never has an entry here and is never nonce-checked.
2083    /// Reserved module ids and the nonce that authorizes their next HELLO.
2084    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2085    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2086    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2087    /// had NO entry and admitted anyone: the reservation protected the nonce
2088    /// holder, not the NAME (found live by CKCRED's canary probe registering
2089    /// against a reserved scratch id).
2090    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2091    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2092    ///
2093    /// This is deliberately in-memory only: subc is state-free across daemon
2094    /// restarts, and the tombstone only explains the hours-after-removal window
2095    /// while this executing daemon is still alive. Do not persist it in a store.
2096    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2097    /// The current launch nonce for every supervised spawn. This is separate from
2098    /// reserved_nonces because consumer route.open attestation applies to all spawned
2099    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2100    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2101    /// Reserved namespace prefixes mapped to the supervised owner module whose
2102    /// current spawn nonce authorizes HELLO claims below the prefix.
2103    ///
2104    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2105    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2106    /// accidental collisions and lower-trust processes from squatting protected
2107    /// namespaces.
2108    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2109    /// Blue/green swaps in progress, by module id. An entry exists from just
2110    /// before the candidate process is spawned until the swap has failed, or
2111    /// has cut over and the old process is gone. While it exists, HELLO for the
2112    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2113    /// consumer attestation accepts both processes' nonces.
2114    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2115    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2116    promotion_observer: PromotionObserverSlot,
2117    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2118    /// this daemon-wide ordering, a rescan could retire or update a module while a
2119    /// concurrent reload still held its old handle and launch specification.
2120    operation_lock: Arc<AsyncMutex<()>>,
2121}
2122
2123/// Told when a swap has promoted its candidate to be the module's active
2124/// registration.
2125///
2126/// An ordinary HELLO runs the control plane's registration side effects (the
2127/// capability cache, the deny census, the requirement recompute) as it
2128/// registers. A swap candidate's HELLO does not, because it is not routable;
2129/// promotion is when those must run instead, and promotion happens in the
2130/// supervisor, which has no other way into the control handler.
2131pub(crate) trait SwapPromotionObserver: Send + Sync {
2132    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2133}
2134
2135/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2136/// control handler) owns this handle, so a strong reference back would be a
2137/// cycle that keeps both alive.
2138#[derive(Clone, Default)]
2139struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2140
2141impl fmt::Debug for PromotionObserverSlot {
2142    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2143        f.write_str("PromotionObserverSlot")
2144    }
2145}
2146
2147/// The nonces of one open swap.
2148#[derive(Debug, Clone)]
2149struct OpenSwap {
2150    /// The launch nonce minted for the candidate process. It is the swap
2151    /// token: the only thing that admits a HELLO into the candidate slot.
2152    candidate_nonce: String,
2153    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2154    /// here because cutover moves the module's recorded spawn nonce to the
2155    /// candidate while the incumbent is still draining and its consumers are
2156    /// still attesting with this one.
2157    incumbent_nonce: Option<String>,
2158    /// Set once a HELLO has been admitted with the swap token, so the token
2159    /// admits one registration and cannot be replayed after cutover empties
2160    /// the candidate slot.
2161    candidate_admitted: bool,
2162}
2163
2164/// What the swap gate says about a HELLO. See
2165/// [`SupervisorHandle::swap_hello_admission`].
2166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2167pub(crate) enum SwapHelloAdmission {
2168    /// No swap is open for the id (or the HELLO carries the incumbent's own
2169    /// nonce); the ordinary gates decide.
2170    NotSwapping,
2171    /// The HELLO carries the swap token: register it into the candidate slot.
2172    Candidate,
2173    /// A swap is open and the HELLO carries a nonce the supervisor did not
2174    /// mint for this id, no nonce, or a token already used.
2175    Refused,
2176}
2177
2178#[derive(Debug, Clone, PartialEq, Eq)]
2179pub(crate) enum ReservedHelloRejection {
2180    Exact {
2181        module_id: String,
2182    },
2183    Prefix {
2184        prefix: String,
2185        owner_module_id: String,
2186    },
2187}
2188
2189/// Identity facts established by the reserved-module policy for one HELLO.
2190#[derive(Debug, Default)]
2191pub(crate) struct ReservedHelloVerification {
2192    pub(crate) nonce_verified: bool,
2193    pub(crate) prefix_owner: Option<String>,
2194}
2195
2196#[derive(Debug)]
2197struct DaemonRestartIntent {
2198    expires: Option<Duration>,
2199    window: Duration,
2200}
2201
2202impl Default for DaemonRestartIntent {
2203    fn default() -> Self {
2204        Self {
2205            expires: None,
2206            window: Duration::from_secs(120),
2207        }
2208    }
2209}
2210
2211impl SupervisorHandle {
2212    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2213        let mut intent = self
2214            .restart_intent
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner());
2217        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2218        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2219    }
2220
2221    #[cfg(feature = "test-support")]
2222    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2223        self.restart_intent
2224            .lock()
2225            .unwrap_or_else(|p| p.into_inner())
2226            .window = window;
2227    }
2228
2229    #[cfg(unix)]
2230    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2231        let expiry = self
2232            .restart_intent
2233            .lock()
2234            .unwrap_or_else(|p| p.into_inner())
2235            .expires
2236            .take();
2237        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2238            RouteCloseReason::Restart
2239        } else {
2240            RouteCloseReason::Disable
2241        }
2242    }
2243
2244    pub fn new() -> Self {
2245        Self::default()
2246    }
2247
2248    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2249        self.spawn_events.snapshot()
2250    }
2251
2252    pub(crate) fn subscribe_spawns(
2253        &self,
2254        connection_id: ConnectionId,
2255        corr: u64,
2256        version: u8,
2257        since: Option<SpawnCursor>,
2258        sink: FrameSink,
2259    ) -> Result<(), SpawnSubscribeRefusal> {
2260        self.spawn_events
2261            .subscribe(connection_id, corr, version, since, sink)
2262    }
2263
2264    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2265        self.spawn_events.cancel(connection_id, corr)
2266    }
2267
2268    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2269        self.spawn_events.remove_connection(connection_id);
2270    }
2271
2272    #[cfg(any(test, feature = "test-support"))]
2273    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2274        assert!(capacity > 0, "spawn event capacity must be non-zero");
2275        self.spawn_events.set_capacity(capacity);
2276    }
2277
2278    #[cfg(any(test, feature = "test-support"))]
2279    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2280        self.spawn_events.subscriber_count()
2281    }
2282
2283    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2284    /// a respawn invalidates stale consumer identities.
2285    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2286        self.spawn_nonces
2287            .lock()
2288            .unwrap_or_else(|poisoned| poisoned.into_inner())
2289            .insert(module_id.to_string(), nonce);
2290    }
2291
2292    /// Record the launch nonce expected from the next HELLO for a reserved module,
2293    /// replacing any prior nonce (a respawn invalidates the previous one).
2294    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2295        self.reserved_nonces
2296            .lock()
2297            .unwrap_or_else(|poisoned| poisoned.into_inner())
2298            .insert(module_id.to_string(), Some(nonce));
2299    }
2300
2301    /// Record namespace prefixes owned by a supervised module.
2302    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2303        let mut owners = self
2304            .reserved_prefix_owners
2305            .lock()
2306            .unwrap_or_else(|poisoned| poisoned.into_inner());
2307        owners.retain(|_, owner| owner != owner_module_id);
2308        for prefix in prefixes {
2309            owners.insert(prefix.clone(), owner_module_id.to_string());
2310        }
2311    }
2312
2313    /// The launch nonce most recently minted for a module's spawn, if any.
2314    #[cfg(test)]
2315    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2316        self.spawn_nonces
2317            .lock()
2318            .unwrap_or_else(|poisoned| poisoned.into_inner())
2319            .get(module_id)
2320            .cloned()
2321    }
2322
2323    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2324        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2325        let spawn_nonce = self
2326            .spawn_nonces
2327            .lock()
2328            .unwrap_or_else(|poisoned| poisoned.into_inner())
2329            .get(&spec.module_id)
2330            .cloned();
2331        let mut reserved_nonces = self
2332            .reserved_nonces
2333            .lock()
2334            .unwrap_or_else(|poisoned| poisoned.into_inner());
2335        if spec.reserved {
2336            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2337            // reserved name whose module has never spawned has no legitimate
2338            // holder, and the entry's absence is what used to leave the name
2339            // open to the first claimant.
2340            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2341        }
2342        drop(reserved_nonces);
2343        // A later unreserved declaration must not silently unreserve an id that
2344        // was retained after its reserved configuration was removed. The explicit
2345        // release ceremony is the only operation that retires that gate.
2346        self.removal_tombstones
2347            .lock()
2348            .unwrap_or_else(|poisoned| poisoned.into_inner())
2349            .remove(&spec.module_id);
2350    }
2351
2352    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2353    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2354    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2355    /// with no matching prefix are always authorized.
2356    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2357        self.reserved_hello_verification(module_id, presented)
2358            .is_ok()
2359    }
2360
2361    /// Whether HELLO passed a nonce check that was actually required by reserved
2362    /// module policy, or the refusal explaining why a required check failed.
2363    pub(crate) fn reserved_hello_verification(
2364        &self,
2365        module_id: &str,
2366        presented: Option<&str>,
2367    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2368        let nonces = self
2369            .reserved_nonces
2370            .lock()
2371            .unwrap_or_else(|poisoned| poisoned.into_inner());
2372        if let Some(expected) = nonces.get(module_id) {
2373            // `None` = reserved with no legitimate holder: refuse every
2374            // presentation, because no process can hold a nonce that was never
2375            // minted. Only a real minted nonce admits, in constant time.
2376            let authorized = match expected {
2377                Some(expected) => {
2378                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2379                }
2380                None => false,
2381            };
2382            if authorized {
2383                return Ok(ReservedHelloVerification {
2384                    nonce_verified: true,
2385                    prefix_owner: None,
2386                });
2387            }
2388            return Err(ReservedHelloRejection::Exact {
2389                module_id: module_id.to_string(),
2390            });
2391        }
2392        drop(nonces);
2393
2394        let matched_prefix = self
2395            .reserved_prefix_owners
2396            .lock()
2397            .unwrap_or_else(|poisoned| poisoned.into_inner())
2398            .iter()
2399            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2400            .max_by_key(|(prefix, _)| prefix.len())
2401            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2402        let Some((prefix, owner_module_id)) = matched_prefix else {
2403            return Ok(ReservedHelloVerification::default());
2404        };
2405
2406        let authorized = presented.is_some_and(|presented| {
2407            self.spawn_nonces
2408                .lock()
2409                .unwrap_or_else(|poisoned| poisoned.into_inner())
2410                .get(&owner_module_id)
2411                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2412                // While the owner is being swapped, children started by
2413                // either of its two processes hold that process's nonce.
2414                || self.swap_nonce_matches(&owner_module_id, presented)
2415        });
2416        if authorized {
2417            Ok(ReservedHelloVerification {
2418                nonce_verified: true,
2419                prefix_owner: Some(owner_module_id),
2420            })
2421        } else {
2422            Err(ReservedHelloRejection::Prefix {
2423                prefix,
2424                owner_module_id,
2425            })
2426        }
2427    }
2428
2429    /// Whether a consumer connection proved it came from a daemon-spawned module.
2430    ///
2431    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2432    /// accepted only for module ids the supervisor has spawned.
2433    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2434        if presented.is_empty() {
2435            return false;
2436        }
2437        let nonces = self
2438            .spawn_nonces
2439            .lock()
2440            .unwrap_or_else(|poisoned| poisoned.into_inner());
2441        let current = nonces
2442            .get(module_id)
2443            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2444        drop(nonces);
2445        // During a swap two processes of the module are alive, and a consumer
2446        // started by either one presents that process's nonce. Accepting only
2447        // the recorded one would fail the incumbent's consumers for the whole
2448        // overlap once cutover moves the record to the candidate.
2449        current || self.swap_nonce_matches(module_id, presented)
2450    }
2451
2452    /// Whether `presented` is either nonce of an open swap for `module_id`.
2453    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2454        let swaps = self
2455            .swaps
2456            .lock()
2457            .unwrap_or_else(|poisoned| poisoned.into_inner());
2458        swaps.get(module_id).is_some_and(|swap| {
2459            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2460                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2461                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2462                })
2463        })
2464    }
2465
2466    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2467    /// Called before the candidate process exists.
2468    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2469        let incumbent_nonce = self
2470            .spawn_nonces
2471            .lock()
2472            .unwrap_or_else(|poisoned| poisoned.into_inner())
2473            .get(module_id)
2474            .cloned();
2475        self.swaps
2476            .lock()
2477            .unwrap_or_else(|poisoned| poisoned.into_inner())
2478            .insert(
2479                module_id.to_string(),
2480                OpenSwap {
2481                    candidate_nonce,
2482                    incumbent_nonce,
2483                    candidate_admitted: false,
2484                },
2485            );
2486    }
2487
2488    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2489    /// the module's recorded one.
2490    pub(crate) fn close_swap(&self, module_id: &str) {
2491        self.swaps
2492            .lock()
2493            .unwrap_or_else(|poisoned| poisoned.into_inner())
2494            .remove(module_id);
2495    }
2496
2497    /// Install the observer told about swap promotions, replacing any earlier
2498    /// one.
2499    pub(crate) fn set_swap_promotion_observer(
2500        &self,
2501        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2502    ) {
2503        *self
2504            .promotion_observer
2505            .0
2506            .lock()
2507            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2508    }
2509
2510    /// Tell the installed observer, if it is still alive, that a swap promoted
2511    /// `registration`.
2512    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2513        let observer = self
2514            .promotion_observer
2515            .0
2516            .lock()
2517            .unwrap_or_else(|poisoned| poisoned.into_inner())
2518            .as_ref()
2519            .and_then(std::sync::Weak::upgrade);
2520        if let Some(observer) = observer {
2521            observer.swap_promoted(registration);
2522        }
2523    }
2524
2525    /// Whether a swap is open for `module_id`.
2526    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2527        self.swaps
2528            .lock()
2529            .unwrap_or_else(|poisoned| poisoned.into_inner())
2530            .contains_key(module_id)
2531    }
2532
2533    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2534    /// respawn would, once cutover has made the candidate the module's process.
2535    /// The swap stays open so the incumbent's nonce keeps attesting until the
2536    /// incumbent has drained and exited.
2537    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2538        let candidate_nonce = self
2539            .swaps
2540            .lock()
2541            .unwrap_or_else(|poisoned| poisoned.into_inner())
2542            .get(module_id)
2543            .map(|swap| swap.candidate_nonce.clone());
2544        let Some(nonce) = candidate_nonce else {
2545            return;
2546        };
2547        self.set_spawn_nonce(module_id, nonce.clone());
2548        if reserved {
2549            self.set_reserved_nonce(module_id, nonce);
2550        }
2551    }
2552
2553    /// The swap gate for a HELLO claiming `module_id`.
2554    ///
2555    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2556    /// presents the candidate nonce, which the reserved gate (holding the
2557    /// incumbent's nonce) would refuse as `reserved_module` before swap
2558    /// admission was ever reached. And it applies to unreserved ids too: for an
2559    /// unreserved id the only thing that ever stopped a second process claiming
2560    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2561    /// refusal a swap lifts for its candidate.
2562    ///
2563    /// The incumbent's own nonce falls through to the ordinary gates, which
2564    /// treat it as they always have (a live incumbent is refused as a
2565    /// duplicate). Anything else while a swap is open is refused, including an
2566    /// absent nonce.
2567    pub(crate) fn swap_hello_admission(
2568        &self,
2569        module_id: &str,
2570        presented: Option<&str>,
2571    ) -> SwapHelloAdmission {
2572        let swaps = self
2573            .swaps
2574            .lock()
2575            .unwrap_or_else(|poisoned| poisoned.into_inner());
2576        let Some(swap) = swaps.get(module_id) else {
2577            return SwapHelloAdmission::NotSwapping;
2578        };
2579        let Some(presented) = presented else {
2580            return SwapHelloAdmission::Refused;
2581        };
2582        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2583            return if swap.candidate_admitted {
2584                SwapHelloAdmission::Refused
2585            } else {
2586                SwapHelloAdmission::Candidate
2587            };
2588        }
2589        if swap
2590            .incumbent_nonce
2591            .as_deref()
2592            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2593        {
2594            return SwapHelloAdmission::NotSwapping;
2595        }
2596        SwapHelloAdmission::Refused
2597    }
2598
2599    /// Record that the swap token has registered a candidate, so it admits no
2600    /// second HELLO.
2601    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2602        if let Some(swap) = self
2603            .swaps
2604            .lock()
2605            .unwrap_or_else(|poisoned| poisoned.into_inner())
2606            .get_mut(module_id)
2607        {
2608            swap.candidate_admitted = true;
2609        }
2610    }
2611
2612    /// Test/support lookup for the current launch nonce of a supervised spawn.
2613    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2614        self.spawn_nonces
2615            .lock()
2616            .unwrap_or_else(|poisoned| poisoned.into_inner())
2617            .get(module_id)
2618            .cloned()
2619    }
2620
2621    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2622    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2623        self.reserved_nonces
2624            .lock()
2625            .unwrap_or_else(|poisoned| poisoned.into_inner())
2626            .get(module_id)
2627            .cloned()
2628            .flatten()
2629    }
2630
2631    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2632        // Normally already marked before the process was spawned; marking here
2633        // too keeps `configured_ids` a superset of the roster for any caller
2634        // that inserts a module directly.
2635        self.mark_configured(module.module_id());
2636        let mut modules = self
2637            .modules
2638            .lock()
2639            .unwrap_or_else(|poisoned| poisoned.into_inner());
2640        modules.insert(module.module_id().to_string(), module)
2641    }
2642
2643    /// Record that the supervisor has taken on `module_id`. Called before the
2644    /// module's first process is spawned, so that by the time that process can
2645    /// register, [`Self::is_configured`] already answers true.
2646    fn mark_configured(&self, module_id: &str) {
2647        self.configured_ids
2648            .lock()
2649            .unwrap_or_else(|poisoned| poisoned.into_inner())
2650            .insert(module_id.to_string());
2651    }
2652
2653    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2654    /// before it was ever put on the roster. A module already on the roster
2655    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2656    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2657        let modules = self
2658            .modules
2659            .lock()
2660            .unwrap_or_else(|poisoned| poisoned.into_inner());
2661        if !modules.contains_key(module_id) {
2662            self.configured_ids
2663                .lock()
2664                .unwrap_or_else(|poisoned| poisoned.into_inner())
2665                .remove(module_id);
2666        }
2667    }
2668
2669    /// Whether `module_id` is a module this daemon supervises: on the roster,
2670    /// or about to be (its process is being spawned right now).
2671    ///
2672    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2673    /// for scopes: a supervised module's process can register and sync before
2674    /// [`Self::get`] can return it, and in that window it is still a module
2675    /// that will sync, not one that never will.
2676    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2677        self.configured_ids
2678            .lock()
2679            .unwrap_or_else(|poisoned| poisoned.into_inner())
2680            .contains(module_id)
2681    }
2682
2683    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2684        let modules = self
2685            .modules
2686            .lock()
2687            .unwrap_or_else(|poisoned| poisoned.into_inner());
2688        modules.get(module_id).cloned()
2689    }
2690
2691    pub(crate) fn record_late_health_answer(
2692        &self,
2693        module_id: &str,
2694        latency_ms: u64,
2695    ) -> Result<bool, SuperviseError> {
2696        let Some(module) = self.get(module_id) else {
2697            return Ok(false);
2698        };
2699        self.host_health.record_late_module(module_id);
2700        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2701            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2702            state.health.last_late_answer_latency_ms = Some(latency_ms);
2703            // A late answer is an answer: the module served the probe, just past
2704            // the deadline. Leaving the miss streak in place while logging
2705            // "proves the module is alive" is how a CPU-starved module that
2706            // answers every probe a few seconds late still marches to the
2707            // threshold and gets killed — the exact kill class `NoAnswer` is
2708            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2709            // is degradation, and degradation reports; it does not restart.
2710            state.health.consecutive_failures = 0;
2711        })?;
2712        Ok(true)
2713    }
2714
2715    pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
2716        self.host_health.spawn_monitor()
2717    }
2718
2719    /// Arm the one-shot marker for the module process that this caller
2720    /// deliberately initiated severance against. Generic connection teardown
2721    /// must not call this:
2722    /// a surviving process would otherwise retain an exemption for a later
2723    /// genuine crash.
2724    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2725        let Some(module) = self.get(module_id) else {
2726            return Ok(false);
2727        };
2728        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2729        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2730            return Ok(false);
2731        };
2732        drop(snapshot);
2733        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2734    }
2735
2736    pub fn list(&self) -> Vec<SupervisedModule> {
2737        let modules = self
2738            .modules
2739            .lock()
2740            .unwrap_or_else(|poisoned| poisoned.into_inner());
2741        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2742        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2743        modules
2744    }
2745
2746    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2747        self.activity_images
2748            .lock()
2749            .unwrap_or_else(|p| p.into_inner())
2750            .remove(module_id);
2751        self.spawn_nonces
2752            .lock()
2753            .unwrap_or_else(|poisoned| poisoned.into_inner())
2754            .remove(module_id);
2755        self.close_swap(module_id);
2756        let mut reserved_nonces = self
2757            .reserved_nonces
2758            .lock()
2759            .unwrap_or_else(|poisoned| poisoned.into_inner());
2760        if reserved_nonces.contains_key(module_id) {
2761            // The old nonce must die with the removed process, but the exact-id
2762            // gate remains until an operator explicitly releases it.
2763            reserved_nonces.insert(module_id.to_string(), None);
2764        }
2765        drop(reserved_nonces);
2766        self.reserved_prefix_owners
2767            .lock()
2768            .unwrap_or_else(|poisoned| poisoned.into_inner())
2769            .retain(|_, owner| owner != module_id);
2770        let removed = self
2771            .modules
2772            .lock()
2773            .unwrap_or_else(|poisoned| poisoned.into_inner())
2774            .remove(module_id);
2775        self.configured_ids
2776            .lock()
2777            .unwrap_or_else(|poisoned| poisoned.into_inner())
2778            .remove(module_id);
2779        removed
2780    }
2781
2782    /// Remember a module removed by a non-preview rescan so route.open can
2783    /// distinguish that intentional removal from an unknown id.
2784    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2785        self.removal_tombstones
2786            .lock()
2787            .unwrap_or_else(|poisoned| poisoned.into_inner())
2788            .insert(module_id.to_string(), unix_ms_now());
2789    }
2790
2791    /// Return how long ago a rescan removed this module in milliseconds.
2792    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2793        self.removal_tombstones
2794            .lock()
2795            .unwrap_or_else(|poisoned| poisoned.into_inner())
2796            .get(module_id)
2797            .copied()
2798            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2799    }
2800
2801    /// Retire a reserved-id gate only after its module has left supervision.
2802    ///
2803    /// A retained gate has no live nonce (`None`), so releasing any other entry
2804    /// would weaken a currently configured or otherwise active reservation.
2805    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2806        if self.get(module_id).is_some() {
2807            return false;
2808        }
2809        let mut reserved_nonces = self
2810            .reserved_nonces
2811            .lock()
2812            .unwrap_or_else(|poisoned| poisoned.into_inner());
2813        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2814            return false;
2815        }
2816        reserved_nonces.remove(module_id);
2817        true
2818    }
2819
2820    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2821        Arc::clone(&self.operation_lock)
2822    }
2823}
2824
2825/// Process supervisor for subc-owned singleton modules.
2826#[derive(Debug, Clone)]
2827pub struct Supervisor {
2828    registry: Arc<Registry>,
2829    restart_policy: RestartPolicy,
2830    drain_timeout: Duration,
2831    connection_file_path: Option<PathBuf>,
2832    capture_logs_dir: Option<PathBuf>,
2833    forwarding: Option<Arc<ForwardingTable>>,
2834    process_liveness: Arc<SupervisorProcessLiveness>,
2835    supervisor_handle: Option<SupervisorHandle>,
2836    health: HealthConfig,
2837    host_health: HostHealth,
2838    daemon_start_clock: crate::clock::StartClock,
2839    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2840    spawn_events: SpawnEventFeed,
2841    provenance_probe: ExecutableIdentityProbe,
2842    /// Every process spawned through this supervisor (and its clones) and not
2843    /// yet reaped, so daemon shutdown can end them.
2844    child_roster: ChildRoster,
2845    #[cfg(target_os = "linux")]
2846    cgroup_placement: Option<subc_cgroup::Placement>,
2847    #[cfg(test)]
2848    test_after_first_spawn: AfterFirstSpawnHook,
2849}
2850
2851/// A roster entry whose monitor and first process have not started yet.
2852/// Keeping the command receiver here lets discovery serve status immediately
2853/// while start/stop requests wait until bootstrap has published the connection file.
2854pub(crate) struct RegisteredConfiguredModule {
2855    spec: ModuleSpec,
2856    runtime: SupervisorRuntimeConfig,
2857    module: SupervisedModule,
2858    commands: mpsc::Receiver<SupervisorCommand>,
2859}
2860
2861/// Test-only hook run on the path that takes on a new module, right after its
2862/// first `spawn_child` returns (the process exists and could already be
2863/// registering) and before that process is handed to the module's supervise
2864/// loop and put on the roster. Lets a test observe what a fast child would see
2865/// in that window without racing a real one.
2866#[cfg(test)]
2867#[derive(Clone, Default)]
2868struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2869
2870#[cfg(test)]
2871type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2872
2873#[cfg(test)]
2874impl fmt::Debug for AfterFirstSpawnHook {
2875    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2876        f.write_str("AfterFirstSpawnHook")
2877    }
2878}
2879
2880#[cfg(test)]
2881impl AfterFirstSpawnHook {
2882    fn run(&self, module_id: &str) {
2883        if let Some(hook) = &self.0 {
2884            hook(module_id);
2885        }
2886    }
2887}
2888
2889impl Supervisor {
2890    #[cfg(test)]
2891    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2892        let supervisor = Self::new(registry, policy);
2893        #[cfg(target_os = "macos")]
2894        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2895        supervisor
2896    }
2897    /// Verify the trampoline once when configured. Missing private OS support
2898    /// refuses every macOS launch by name but does not stop the daemon's control
2899    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2900    /// entry point; the library must not exec an arbitrary hosting program.
2901    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2902        let path = path.into();
2903        #[cfg(target_os = "macos")]
2904        {
2905            let result = probe_privacy_trampoline(&path).map(|()| path);
2906            if let Err(cause) = &result {
2907                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2908            }
2909            self.child_roster.set_privacy_trampoline(result);
2910        }
2911        #[cfg(not(target_os = "macos"))]
2912        let _ = path;
2913        self
2914    }
2915    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2916    /// capability-requirement log to read.
2917    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2918        self.child_roster.shutdown_flag()
2919    }
2920
2921    /// The first step of an announced daemon shutdown, before the notice and
2922    /// before any connection is closed.
2923    ///
2924    /// Sets the daemon-shutdown flag first: from here on no module is
2925    /// respawned (crash restart, operator restart, or swap), and every child
2926    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2927    /// the module exits on the EOF this shutdown gives it or is signalled by a
2928    /// service manager that kills the whole cgroup. Then writes the journal's
2929    /// shutdown marker, which records the instant and closes this daemon
2930    /// incarnation's stretch of the journal.
2931    #[cfg(unix)]
2932    pub(crate) fn begin_daemon_shutdown(&self) {
2933        self.child_roster.close();
2934        if let Some(journal) = &self.terminal_journal {
2935            journal.stamp_shutdown();
2936        }
2937    }
2938
2939    /// Announce a cut while established connections can still carry replies.
2940    /// These budgets bound the attempt to deliver notices and the wait for
2941    /// routes to drain; neither guarantees delivery or child completion, and
2942    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2943    #[cfg(unix)]
2944    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2945        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2946        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2947        let Some(forwarding) = &self.forwarding else {
2948            return Ok(());
2949        };
2950        let notice_started = Instant::now();
2951        let modules = forwarding
2952            .begin_daemon_drain()
2953            .map_err(SuperviseError::Forwarding)?;
2954        let deadline_ms =
2955            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2956        let shutdown_reason = self
2957            .supervisor_handle
2958            .as_ref()
2959            .map(SupervisorHandle::take_daemon_shutdown_reason)
2960            .unwrap_or(RouteCloseReason::Disable);
2961        let mut modules_told = 0;
2962        let mut notices = tokio::task::JoinSet::new();
2963        let mut drains = Vec::new();
2964        for connection in forwarding
2965            .module_connections()
2966            .map_err(SuperviseError::Forwarding)?
2967        {
2968            let target = ModuleDrainTarget {
2969                endpoint: connection.endpoint,
2970                sink: connection.sink,
2971                negotiated_ver: connection.negotiated_ver,
2972                abandoned_bindings: Vec::new(),
2973                excluded_subscriptions: 0,
2974            };
2975            notices.spawn(async move {
2976                (
2977                    true,
2978                    send_module_draining(
2979                        &connection.module_id,
2980                        shutdown_reason,
2981                        deadline_ms,
2982                        &target,
2983                        true,
2984                    )
2985                    .await,
2986                )
2987            });
2988        }
2989        for (module_id, endpoint) in modules {
2990            let Some(target) = forwarding
2991                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2992                .map_err(SuperviseError::Forwarding)?
2993            else {
2994                continue;
2995            };
2996            let routes = forwarding
2997                .endpoint_routes(target.endpoint)
2998                .map_err(SuperviseError::Forwarding)?;
2999            // Only an announced restart promises that modules are coming back.
3000            // Client route notices retain their existing reconnect semantics.
3001            let mut recipients = Vec::new();
3002            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3003            for route in routes {
3004                let client = route.goodbye_target;
3005                if let Some((_, channels)) = clients
3006                    .iter_mut()
3007                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3008                {
3009                    channels.push(client.channel);
3010                } else {
3011                    let channel = client.channel;
3012                    clients.push((client, vec![channel]));
3013                }
3014            }
3015            for (client, mut channels) in clients {
3016                channels.sort_unstable();
3017                channels.dedup();
3018                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3019                    module_id: module_id.clone(),
3020                    channels,
3021                    reason: RouteCloseReason::Restart,
3022                })
3023                .expect("route closing serializes");
3024                recipients.push((client.sink, client.negotiated_ver, closing));
3025            }
3026            for (sink, version, body) in recipients {
3027                notices.spawn(async move {
3028                    let frame = Frame::build_with_version(
3029                        version,
3030                        FrameType::Push,
3031                        control_flags(),
3032                        0,
3033                        0,
3034                        0,
3035                        body,
3036                    )
3037                    .expect("bounded lifecycle notice frame builds");
3038                    (false, sink.send_flushed(frame).await.is_ok())
3039                });
3040            }
3041            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3042            drains.push((module_id, target.endpoint, gauges));
3043        }
3044        // A quiet forwarding table is not proof that queued notices reached the
3045        // socket. Wait for writer flush acknowledgements before testing quiescence.
3046        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3047        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3048            match result {
3049                Ok((true, true)) => modules_told += 1,
3050                Ok((false, true)) => {}
3051                result => warn!(?result, "daemon shutdown notice delivery failed"),
3052            }
3053        }
3054        notices.abort_all();
3055        info!(
3056            ?shutdown_reason,
3057            modules_told, "daemon shutdown module notices delivered"
3058        );
3059        let notice_elapsed = notice_started.elapsed();
3060        let drain_started = Instant::now();
3061        let deadline = drain_started + DRAIN_BUDGET;
3062        let mut waits = tokio::task::JoinSet::new();
3063        for (module_id, endpoint, gauges) in drains {
3064            let forwarding = Arc::clone(forwarding);
3065            let mut runtime = self.runtime_config();
3066            runtime.health.cadence = Duration::from_millis(100);
3067            waits.spawn(async move {
3068                let wait_result = wait_for_forwarding_quiescence(
3069                    &forwarding,
3070                    &module_id,
3071                    &runtime,
3072                    endpoint,
3073                    deadline,
3074                    &gauges,
3075                    DrainScope::Endpoint(endpoint),
3076                    &mut DrainGaugeTrace::new(&gauges),
3077                )
3078                .await;
3079                // Store the measurement before the task returns: quiescence
3080                // may be reached just as the deadline stops the join loop from
3081                // collecting results. Unfinished or failed waits report false.
3082                forwarding
3083                    .record_daemon_drain_result(
3084                        endpoint,
3085                        drained_after_quiescence_wait(&wait_result),
3086                    )
3087                    .map_err(SuperviseError::Forwarding)?;
3088                wait_result
3089            });
3090        }
3091        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3092            if !matches!(result, Ok(Ok(true))) {
3093                warn!(?result, "daemon shutdown drain did not reach quiescence");
3094            }
3095        }
3096        // Measure only the forwarding drain, not child teardown or the caller's
3097        // signal delivery and process-reaping delays.
3098        info!(
3099            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3100            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3101            "daemon shutdown forwarding drain complete"
3102        );
3103        Ok(())
3104    }
3105
3106    /// The last step of a graceful daemon shutdown, after the notice and the
3107    /// drain: send every registered module a module GOODBYE, the same planned
3108    /// stop signal `ck module stop` gives, then close every connection so each
3109    /// subc module sees EOF and starts its own teardown, then end every
3110    /// supervised child that has not exited
3111    /// by its own deadline (its drain budget, capped). Modules lead their own
3112    /// process groups, so a
3113    /// service manager's group kill no longer reaches them; without this a
3114    /// child that does not stop on EOF (every `protocol: "none"` child, which
3115    /// has no connection) would outlive the daemon. Every wait is bounded (see
3116    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3117    #[cfg(unix)]
3118    pub(crate) async fn end_children_for_daemon_shutdown(
3119        &self,
3120        already_escalated: bool,
3121        escalate: impl std::future::Future<Output = ()>,
3122    ) {
3123        tokio::pin!(escalate);
3124        let mut escalated = already_escalated;
3125        if let Some(forwarding) = &self.forwarding {
3126            let reason = CloseReason::new(
3127                "daemon_shutdown",
3128                "the daemon is exiting after its shutdown notice and drain",
3129            );
3130            if escalated {
3131                // The operator asked to stop waiting: queue the GOODBYEs but
3132                // do not wait for them to be written.
3133                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3134            } else {
3135                tokio::select! {
3136                    biased;
3137                    _ = escalate.as_mut() => {
3138                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3139                        escalated = true;
3140                    }
3141                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3142                }
3143            }
3144            let closed = forwarding.close_all_connections(&reason);
3145            debug!(closed, "closed established connections for daemon shutdown");
3146        }
3147        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3148        // already completed and must not be polled again; the child shutdown
3149        // wait is told it is escalated and gets a future that never fires.
3150        let escalated_here = escalated && !already_escalated;
3151        let remaining_escalate = async move {
3152            if escalated_here {
3153                std::future::pending::<()>().await;
3154            } else {
3155                escalate.await;
3156            }
3157        };
3158        crate::child_roster::end_children_for_daemon_shutdown(
3159            &self.child_roster,
3160            escalated,
3161            remaining_escalate,
3162        )
3163        .await;
3164    }
3165
3166    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3167        Self {
3168            registry,
3169            restart_policy,
3170            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3171            connection_file_path: None,
3172            capture_logs_dir: None,
3173            forwarding: None,
3174            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3175            supervisor_handle: None,
3176            health: HealthConfig::default(),
3177            host_health: HostHealth::default(),
3178            daemon_start_clock: crate::clock::StartClock::capture(),
3179            terminal_journal: None,
3180            spawn_events: SpawnEventFeed::default(),
3181            provenance_probe: ExecutableIdentityProbe::default(),
3182            child_roster: ChildRoster::default(),
3183            #[cfg(target_os = "linux")]
3184            cgroup_placement: None,
3185            #[cfg(test)]
3186            test_after_first_spawn: AfterFirstSpawnHook::default(),
3187        }
3188    }
3189
3190    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3191        self.drain_timeout = drain_timeout;
3192        self
3193    }
3194
3195    /// The drain budget a module gets when its config entry sets none.
3196    pub(crate) fn default_drain_timeout(&self) -> Duration {
3197        self.drain_timeout
3198    }
3199
3200    pub fn with_process_liveness(
3201        mut self,
3202        process_liveness: Arc<SupervisorProcessLiveness>,
3203    ) -> Self {
3204        self.process_liveness = process_liveness;
3205        self
3206    }
3207
3208    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3209        self.connection_file_path = Some(connection_file_path.into());
3210        self
3211    }
3212
3213    /// Enables daemon-owned capture files for supervised stdout and stderr.
3214    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3215        self.capture_logs_dir = Some(logs_dir.into());
3216        self
3217    }
3218
3219    /// Names this daemon lifetime in spawn events, independently of whether a
3220    /// terminal journal is configured.
3221    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3222        // A millisecond start stamp can repeat after clock rollback or a rapid
3223        // restart. Use the connection file's random daemon_id instead: it already
3224        // identifies this daemon lifetime independently of the wall clock.
3225        self.spawn_events.configure_incarnation(daemon_incarnation);
3226        self
3227    }
3228
3229    /// Enables best-effort history shared by every supervised module. Without
3230    /// it, terminal history is kept only in each module's in-memory ring.
3231    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3232        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3233        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3234            path,
3235            daemon_incarnation,
3236        )));
3237        this
3238    }
3239
3240    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3241        self.child_roster.counters = forwarding.counters();
3242        self.forwarding = Some(forwarding);
3243        self
3244    }
3245
3246    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3247        self.host_health = supervisor_handle.host_health.clone();
3248        self.spawn_events = supervisor_handle.spawn_events.clone();
3249        self.supervisor_handle = Some(supervisor_handle);
3250        self
3251    }
3252
3253    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3254        self.health = health;
3255        self
3256    }
3257
3258    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3259    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3260    /// record is kept.
3261    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3262        self.child_roster.record_to(path.into());
3263        self
3264    }
3265
3266    #[cfg(target_os = "linux")]
3267    pub fn with_cgroup_placement(
3268        mut self,
3269        cgroup_placement: Option<subc_cgroup::Placement>,
3270    ) -> Self {
3271        self.child_roster
3272            .set_cgroup_placement(cgroup_placement.clone());
3273        self.cgroup_placement = cgroup_placement;
3274        self
3275    }
3276
3277    /// Spawn `spec.program` and start monitoring it.
3278    ///
3279    /// The child is expected to parse `--subc <connection-file-path>`, read the
3280    /// TCP+key connection file, authenticate to the already-running listener, and
3281    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3282    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3283        validate_spec(&spec)?;
3284        self.establish_identity(&spec);
3285
3286        let runtime = self.runtime_config();
3287        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3288        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3289        let registration_watch = watch_registration(&self.registry, &spec);
3290        let spawned = spawn_child(
3291            &spec,
3292            runtime.connection_file_path.as_deref(),
3293            self.supervisor_handle.as_ref(),
3294            &runtime.stderr_ring,
3295            runtime.capture_logs_dir.as_deref(),
3296            &runtime.child_roster,
3297            #[cfg(target_os = "linux")]
3298            runtime.cgroup_placement.as_ref(),
3299        );
3300        #[cfg(test)]
3301        self.test_after_first_spawn.run(&spec.module_id);
3302        let mut child = match spawned {
3303            Ok(child) => child,
3304            Err(err) => {
3305                // Unlike the configured paths, a failed `spawn` leaves nothing
3306                // on the roster, so the module must not stay marked configured.
3307                self.abandon_unrostered(&spec.module_id);
3308                return Err(err);
3309            }
3310        };
3311        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3312        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3313        if let Some(spawn) = activity_spawn {
3314            spawn.complete(&snapshot);
3315        }
3316
3317        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3318    }
3319
3320    /// Make `spec`'s module count as configured, with its identity gates
3321    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3322    /// exists.
3323    ///
3324    /// Every path that takes on a new module calls this before `spawn_child`.
3325    /// The order is the point: the child can connect, register, sync its
3326    /// scopes and ask about them as soon as it is spawned, and the module is
3327    /// only put on the roster after `spawn_child` returns. Were the mark set
3328    /// with the roster entry, a fast child would see its own owner reported
3329    /// as not configured, and a scoped `route.open` in that window would be
3330    /// refused as terminal `scope_not_live` ("will never sync") instead of
3331    /// retryable `scope_not_synced`.
3332    fn establish_identity(&self, spec: &ModuleSpec) {
3333        if let Some(supervisor_handle) = &self.supervisor_handle {
3334            supervisor_handle.apply_identity_configuration(spec);
3335            supervisor_handle.mark_configured(&spec.module_id);
3336        }
3337    }
3338
3339    /// Take back [`Self::establish_identity`]'s configured mark when the
3340    /// module will not be put on the roster after all.
3341    fn abandon_unrostered(&self, module_id: &str) {
3342        if let Some(supervisor_handle) = &self.supervisor_handle {
3343            supervisor_handle.unmark_configured_unless_rostered(module_id);
3344        }
3345    }
3346
3347    /// Record a freshly spawned first process as running. On failure the
3348    /// module never reaches the roster, so its configured mark is taken back.
3349    fn mark_first_process_running(
3350        &self,
3351        spec: &ModuleSpec,
3352        runtime: &SupervisorRuntimeConfig,
3353        snapshot: &SharedSnapshot,
3354        child: &mut SupervisedChild,
3355    ) -> Result<(), SuperviseError> {
3356        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3357            containment::terminate_blocking(&runtime.child_roster.containment);
3358            self.abandon_unrostered(&spec.module_id);
3359            return Err(err);
3360        }
3361        self.process_liveness
3362            .track(spec.module_id.clone(), Arc::clone(snapshot));
3363        Ok(())
3364    }
3365
3366    /// Start supervising a module declared in daemon configuration.
3367    ///
3368    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3369    /// failures in the supervisor handle so operator-facing `supervisor.list`
3370    /// reflects every configured module while daemon startup continues.
3371    pub fn supervise_configured(
3372        &self,
3373        spec: ModuleSpec,
3374        enabled: bool,
3375    ) -> Result<SupervisedModule, SuperviseError> {
3376        self.supervise_configured_requested(
3377            spec,
3378            enabled,
3379            self.health.clone(),
3380            None,
3381            self.restart_policy,
3382            DEFAULT_REGISTER_TIMEOUT,
3383            None,
3384        )
3385    }
3386
3387    /// Supervise a configured module with its own health, drain, and crash
3388    /// budget. The restart policy is per-module because the config file is:
3389    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3390    /// module that is expensive to restart should not be forced onto the same
3391    /// budget as one that is cheap.
3392    pub fn supervise_configured_with_health(
3393        &self,
3394        spec: ModuleSpec,
3395        enabled: bool,
3396        health: HealthConfig,
3397        drain_timeout_ms: Option<u64>,
3398        restart_policy: RestartPolicy,
3399    ) -> Result<SupervisedModule, SuperviseError> {
3400        self.supervise_configured_requested(
3401            spec,
3402            enabled,
3403            health,
3404            drain_timeout_ms,
3405            restart_policy,
3406            DEFAULT_REGISTER_TIMEOUT,
3407            None,
3408        )
3409    }
3410
3411    #[allow(clippy::too_many_arguments)]
3412    pub(crate) fn supervise_configured_requested(
3413        &self,
3414        spec: ModuleSpec,
3415        enabled: bool,
3416        health: HealthConfig,
3417        drain_timeout_ms: Option<u64>,
3418        restart_policy: RestartPolicy,
3419        register_timeout: Duration,
3420        requester: Option<Principal>,
3421    ) -> Result<SupervisedModule, SuperviseError> {
3422        let registered = self.register_configured_with_health(
3423            spec,
3424            enabled,
3425            health,
3426            drain_timeout_ms,
3427            restart_policy,
3428            register_timeout,
3429        )?;
3430        let daemon_initiated = requester.is_none();
3431        self.start_registered_configured(registered, requester, daemon_initiated)
3432    }
3433
3434    /// Record a configured module as starting or disabled, without launching a
3435    /// process or running its monitor. Bootstrap registers the entire roster
3436    /// before publishing discovery, then activates it once children can read
3437    /// the connection file passed in their `--subc` argument.
3438    pub(crate) fn register_configured_with_health(
3439        &self,
3440        spec: ModuleSpec,
3441        enabled: bool,
3442        health: HealthConfig,
3443        drain_timeout_ms: Option<u64>,
3444        restart_policy: RestartPolicy,
3445        register_timeout: Duration,
3446    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3447        validate_spec(&spec)?;
3448        self.establish_identity(&spec);
3449
3450        let mut runtime = self.runtime_config();
3451        runtime.health = health;
3452        runtime.restart_policy = restart_policy;
3453        runtime.register_timeout = register_timeout;
3454        if let Some(ms) = drain_timeout_ms {
3455            runtime.drain_timeout = Duration::from_millis(ms);
3456            *runtime
3457                .effective_drain_timeout
3458                .lock()
3459                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3460        }
3461        let snapshot = if enabled {
3462            SupervisorSnapshot::starting()
3463        } else {
3464            SupervisorSnapshot::disabled()
3465        };
3466        let snapshot = {
3467            let mut snapshot = snapshot;
3468            if enabled {
3469                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3470                {
3471                    snapshot.waiting_for_previous_pid = Some(pid);
3472                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3473                }
3474            }
3475            snapshot
3476        };
3477        let (module, commands) =
3478            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3479        Ok(RegisteredConfiguredModule {
3480            spec,
3481            runtime,
3482            module,
3483            commands,
3484        })
3485    }
3486
3487    /// Activate a previously registered module. Queued control requests are
3488    /// served by the monitor after the initial configured spawn has settled.
3489    pub(crate) fn start_registered_configured(
3490        &self,
3491        registered: RegisteredConfiguredModule,
3492        requester: Option<Principal>,
3493        daemon_initiated: bool,
3494    ) -> Result<SupervisedModule, SuperviseError> {
3495        let RegisteredConfiguredModule {
3496            spec,
3497            mut runtime,
3498            module,
3499            commands,
3500        } = registered;
3501        runtime.start_requester = requester;
3502        runtime.recovery_is_daemon = daemon_initiated;
3503        let snapshot = &module.inner.snapshot;
3504        let should_spawn = {
3505            let state = lock_snapshot(snapshot)?;
3506            state.enabled && state.waiting_for_previous_pid.is_none()
3507        };
3508        let child = if should_spawn {
3509            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3510            let registration_watch = watch_registration(&self.registry, &spec);
3511            let spawned = spawn_child(
3512                &spec,
3513                runtime.connection_file_path.as_deref(),
3514                self.supervisor_handle.as_ref(),
3515                &runtime.stderr_ring,
3516                runtime.capture_logs_dir.as_deref(),
3517                &runtime.child_roster,
3518                #[cfg(target_os = "linux")]
3519                runtime.cgroup_placement.as_ref(),
3520            );
3521            #[cfg(test)]
3522            self.test_after_first_spawn.run(&spec.module_id);
3523            match spawned {
3524                Ok(mut child) => {
3525                    arm_registration_deadline(
3526                        &mut child,
3527                        runtime.register_timeout,
3528                        registration_watch,
3529                    );
3530                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3531                    if let Some(spawn) = activity_spawn {
3532                        spawn.complete(snapshot);
3533                    }
3534                    Some(child)
3535                }
3536                Err(err) => {
3537                    if runtime.health.critical {
3538                        error!(
3539                            module_id = %spec.module_id,
3540                            program = %spec.program.display(),
3541                            error = %err,
3542                            "critical configured module failed to spawn; marking failed and alerting"
3543                        );
3544                    } else {
3545                        error!(
3546                            module_id = %spec.module_id,
3547                            program = %spec.program.display(),
3548                            error = %err,
3549                            "configured module failed to spawn; marking failed and continuing"
3550                        );
3551                    }
3552                    fail_snapshot(
3553                        snapshot,
3554                        Some(&spec.module_id),
3555                        None,
3556                        &runtime.failure_context(daemon_initiated),
3557                    );
3558                    None
3559                }
3560            }
3561        } else {
3562            None
3563        };
3564        self.start_monitor(spec, runtime, &module, child, commands);
3565        Ok(module)
3566    }
3567
3568    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3569        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3570        SupervisorRuntimeConfig {
3571            registry: self.registry.clone(),
3572            recovery_is_daemon: true,
3573            start_requester: None,
3574            pending_activity_restart: false,
3575            provenance_probe: self.provenance_probe.clone(),
3576            scheduled_respawn: Arc::default(),
3577            deferred_reload_reply: Arc::default(),
3578            restart_policy: self.restart_policy,
3579            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3580            drain_timeout: self.drain_timeout,
3581            // Shared with this module's roster copy: daemon shutdown waits on
3582            // each child for the module's own drain budget, as resolved now.
3583            child_roster: self
3584                .child_roster
3585                .for_module(Arc::clone(&effective_drain_timeout)),
3586            effective_drain_timeout,
3587            default_drain_timeout: self.drain_timeout,
3588            health: self.health.clone(),
3589            host_health: self.host_health.clone(),
3590            connection_file_path: self.connection_file_path.clone(),
3591            capture_logs_dir: self.capture_logs_dir.clone(),
3592            forwarding: self.forwarding.clone(),
3593            supervisor_handle: self.supervisor_handle.clone(),
3594            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3595            terminal_ring: Arc::new(Mutex::new(
3596                TerminalRing::new(
3597                    TerminalRingConfig::default(),
3598                    self.daemon_start_clock.started_at_ms(),
3599                )
3600                .with_start_clock(self.daemon_start_clock)
3601                .with_journal(self.terminal_journal.clone())
3602                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3603            )),
3604            spawn_events: self.spawn_events.clone(),
3605            #[cfg(target_os = "linux")]
3606            cgroup_placement: self.cgroup_placement.clone(),
3607            #[cfg(test)]
3608            test_seed_stale_facts_before_enable_spawn: false,
3609            #[cfg(test)]
3610            test_reload_exit_record_gate: None,
3611        }
3612    }
3613
3614    fn supervised_module(
3615        &self,
3616        spec: ModuleSpec,
3617        runtime: SupervisorRuntimeConfig,
3618        snapshot: SharedSnapshot,
3619        child: Option<SupervisedChild>,
3620    ) -> SupervisedModule {
3621        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3622        self.start_monitor(spec, runtime, &module, child, commands);
3623        module
3624    }
3625
3626    fn register_module(
3627        &self,
3628        spec: &ModuleSpec,
3629        runtime: &SupervisorRuntimeConfig,
3630        snapshot: SharedSnapshot,
3631    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3632        runtime.host_health.track(spec, &snapshot);
3633        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3634            spec: spec.clone(),
3635            health: runtime.health.clone(),
3636        }));
3637        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3638        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3639        // The module's OWN policy, which may be its per-module config rather than
3640        // the supervisor-wide one; status must report the budget the supervise
3641        // loop actually enforces.
3642        let restart_policy = runtime.restart_policy;
3643        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3644        let (tx, rx) = mpsc::channel(4);
3645        let module_id = spec.module_id.clone();
3646        let module = SupervisedModule {
3647            inner: Arc::new(SupervisedModuleInner {
3648                module_id: module_id.clone(),
3649                registry: Arc::clone(&self.registry),
3650                snapshot,
3651                configuration,
3652                stderr_ring,
3653                terminal_ring,
3654                commands: tx,
3655                monitor: Mutex::new(None),
3656                restart_policy,
3657                effective_drain_timeout,
3658                provenance_probe: self.provenance_probe.clone(),
3659                retained: runtime.child_roster.retained_view(),
3660            }),
3661        };
3662        // Identity gates and the configured mark must precede the roster entry,
3663        // so a client never sees a module without its admission policy.
3664        if let Some(supervisor_handle) = &self.supervisor_handle {
3665            supervisor_handle.insert(module.clone());
3666        }
3667        (module, rx)
3668    }
3669
3670    fn start_monitor(
3671        &self,
3672        spec: ModuleSpec,
3673        runtime: SupervisorRuntimeConfig,
3674        module: &SupervisedModule,
3675        child: Option<SupervisedChild>,
3676        commands: mpsc::Receiver<SupervisorCommand>,
3677    ) {
3678        let monitor = tokio::spawn(supervise_loop(
3679            spec,
3680            runtime,
3681            Arc::clone(&self.registry),
3682            Arc::clone(&self.process_liveness),
3683            Arc::clone(&module.inner.snapshot),
3684            child,
3685            commands,
3686        ));
3687        *module
3688            .inner
3689            .monitor
3690            .lock()
3691            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3692    }
3693}
3694
3695impl Default for Supervisor {
3696    fn default() -> Self {
3697        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3698    }
3699}
3700
3701/// Handle to one supervised child process.
3702#[derive(Clone)]
3703pub struct SupervisedModule {
3704    inner: Arc<SupervisedModuleInner>,
3705}
3706
3707struct SupervisedModuleInner {
3708    retained: crate::child_roster::RetainedView,
3709    module_id: String,
3710    registry: Arc<Registry>,
3711    snapshot: SharedSnapshot,
3712    configuration: Arc<Mutex<SupervisedConfiguration>>,
3713    stderr_ring: Arc<Mutex<StderrRing>>,
3714    terminal_ring: Arc<Mutex<TerminalRing>>,
3715    commands: mpsc::Sender<SupervisorCommand>,
3716    monitor: Mutex<Option<JoinHandle<()>>>,
3717    /// Copied from the supervisor's runtime config at spawn so `status()` can
3718    /// report the restart budget without reaching back into the supervisor. The
3719    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3720    restart_policy: RestartPolicy,
3721    effective_drain_timeout: Arc<Mutex<Duration>>,
3722    provenance_probe: ExecutableIdentityProbe,
3723}
3724
3725impl fmt::Debug for SupervisedModule {
3726    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3727        f.debug_struct("SupervisedModule")
3728            .field("module_id", &self.inner.module_id)
3729            .field("status", &self.status())
3730            .finish_non_exhaustive()
3731    }
3732}
3733
3734impl SupervisedModule {
3735    pub fn module_id(&self) -> &str {
3736        &self.inner.module_id
3737    }
3738
3739    /// Test-only: put one probe miss on the streak, the way
3740    /// `handle_health_probe_failure` does, so tests can assert what a later
3741    /// event does to the streak without driving the whole probe loop.
3742    #[cfg(test)]
3743    pub(crate) fn record_health_probe_failure_for_test(
3744        &self,
3745        detail: &str,
3746    ) -> Result<(), SuperviseError> {
3747        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3748            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3749            state.health.detail = Some(detail.to_string());
3750        })
3751    }
3752
3753    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3754        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3755    }
3756
3757    pub(crate) fn registration_end_reason(
3758        &self,
3759    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3760        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3761        Ok(match snapshot.state {
3762            ModuleState::Draining if snapshot.draining_to_replace => {
3763                Some(RegistrationEndReason::SupervisorRestart)
3764            }
3765            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3766            ModuleState::Restarting if snapshot.draining_to_replace => {
3767                Some(RegistrationEndReason::SupervisorRestart)
3768            }
3769            _ => None,
3770        })
3771    }
3772
3773    /// The module's retained stderr, newest lines last.
3774    ///
3775    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3776    /// module, `supervisor.list` renders every module, and putting it in the
3777    /// shared snapshot would make each status read carry a payload almost nobody
3778    /// asked for. Callers that want the text ask for it.
3779    pub fn stderr_tail(
3780        &self,
3781        max_lines: Option<usize>,
3782        max_bytes: Option<usize>,
3783    ) -> StderrTailSnapshot {
3784        self.inner
3785            .stderr_ring
3786            .lock()
3787            .unwrap_or_else(|poisoned| poisoned.into_inner())
3788            .snapshot(max_lines, max_bytes)
3789    }
3790
3791    /// The module's bounded terminal history, oldest retained exit first.
3792    ///
3793    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3794    /// daemon whose in-memory history was necessarily reset.
3795    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3796        self.inner
3797            .terminal_ring
3798            .lock()
3799            .unwrap_or_else(|poisoned| poisoned.into_inner())
3800            .snapshot()
3801    }
3802
3803    /// Retained observations from the current ring and all journal generations.
3804    ///
3805    /// Blocking: this reads the journal files. Async callers use
3806    /// `Self::read_durable_terminal_history`.
3807    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3808        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3809    }
3810
3811    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3812    /// read (up to every retained generation) never occupies a runtime worker.
3813    /// Fails only if the blocking task could not finish (runtime shutdown or a
3814    /// panic in the read).
3815    pub(crate) async fn read_durable_terminal_history(
3816        &self,
3817    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3818        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3819        let module_id = self.inner.module_id.clone();
3820        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3821            .await
3822    }
3823
3824    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3825        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3826            .map(|(status, _)| status)
3827    }
3828
3829    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3830        let state = lock_snapshot(&self.inner.snapshot)?;
3831        Ok(state
3832            .failed_retry
3833            .pending
3834            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3835            .map(|retry| {
3836                unix_ms_now().saturating_add(
3837                    u64::try_from(
3838                        retry
3839                            .deadline
3840                            .saturating_duration_since(Instant::now())
3841                            .as_millis(),
3842                    )
3843                    .unwrap_or(u64::MAX),
3844                )
3845            }))
3846    }
3847
3848    pub(crate) fn record_deliberate_severance(
3849        &self,
3850        identity: ProcessIdentity,
3851    ) -> Result<bool, SuperviseError> {
3852        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3853        if snapshot.pid != Some(identity.pid)
3854            || snapshot.process_start_time != Some(identity.start_time)
3855        {
3856            return Ok(false);
3857        }
3858        snapshot.deliberate_severance = Some(identity);
3859        Ok(true)
3860    }
3861
3862    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3863    ///
3864    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3865    /// does not produce reader-observability logs.
3866    pub(crate) fn status_for_control(
3867        &self,
3868        caller: &'static str,
3869    ) -> Result<ModuleStatus, SuperviseError> {
3870        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3871            .map(|(status, _)| status)
3872    }
3873
3874    fn status_with_snapshot_lock(
3875        &self,
3876        snapshot: &SharedSnapshot,
3877        caller: Option<&'static str>,
3878    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3879        let mut guard = match caller {
3880            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3881            None => lock_snapshot(snapshot)?,
3882        };
3883        // Read the budget through the pruning path so a reader sees the same
3884        // in-window count the restart decision would use, not a stale total.
3885        let restart_count =
3886            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3887        let snapshot = guard.clone();
3888        drop(guard);
3889        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3890            SuperviseError::StatePoisoned {
3891                module_id: Some(self.inner.module_id.clone()),
3892            }
3893        })?;
3894        let registration_active = self
3895            .inner
3896            .registry
3897            .get_module(&self.inner.module_id)
3898            .map_err(SuperviseError::Registry)?
3899            .is_some();
3900        let protocol = snapshot
3901            .spawned_protocol
3902            .unwrap_or(self.declared_protocol()?);
3903        let running_process =
3904            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3905        // Registration is the difference between the two protocols and the only
3906        // one: a subc module that has not registered cannot serve a request even
3907        // though its process is up, and a `none` module never registers at all,
3908        // so requiring it there would pin `live` to false for the whole life of
3909        // a perfectly healthy process.
3910        let live = match protocol {
3911            ModuleProtocol::Subc => running_process && registration_active,
3912            ModuleProtocol::None => running_process,
3913        };
3914        // Read only the retained cache: a status read must not resend signals,
3915        // rewrite the record, or keep the containment resource alive.
3916        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3917        let carry_configured = self
3918            .inner
3919            .configuration
3920            .lock()
3921            .map_err(|_| SuperviseError::StatePoisoned {
3922                module_id: Some(self.inner.module_id.clone()),
3923            })?
3924            .spec
3925            .carry_containment;
3926        let recovery_block = carries_containment(
3927            carry_configured,
3928            self.inner
3929                .retained
3930                .has_carried_container(&self.inner.module_id),
3931        )
3932        .then(|| self.inner.retained.recovery_block())
3933        .flatten();
3934        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3935        let pid = snapshot.reported_pid();
3936        let mut health = snapshot.health;
3937        if let Some(detail) = recovery_block
3938            .as_ref()
3939            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3940        {
3941            health.detail = Some(match health.detail {
3942                Some(previous) => format!("{previous}; {detail}"),
3943                None => detail.clone(),
3944            });
3945        }
3946
3947        Ok((
3948            ModuleStatus {
3949                waiting_for_previous_pid: retained
3950                    .as_ref()
3951                    .map(|(pid, _)| *pid)
3952                    .filter(|pid| *pid > 0)
3953                    .or(snapshot.waiting_for_previous_pid),
3954                waiting_for_previous_detail: recovery_block
3955                    .or_else(|| retained.map(|(_, detail)| detail))
3956                    .or(snapshot.waiting_for_previous_detail),
3957                module_id: self.inner.module_id.clone(),
3958                launch_nonce_source,
3959                state: snapshot.state,
3960                enabled: snapshot.enabled,
3961                process_alive: snapshot.process_alive,
3962                registration_active,
3963                protocol,
3964                live,
3965                restart_count,
3966                lifetime_restarts: snapshot.lifetime_restarts,
3967                spawn_generation: snapshot.spawn_generation,
3968                max_restarts: self.inner.restart_policy.max_restarts,
3969                restart_window: self.inner.restart_policy.window,
3970                drain_timeout,
3971                restart_backoff: self.inner.restart_policy.backoff,
3972                restart_max_backoff: self.inner.restart_policy.max_backoff,
3973                pid,
3974                spawned_at_ms: snapshot.spawned_at_ms,
3975                spawned_from: snapshot.spawned_from,
3976                launch_spec_digest: snapshot.launch_spec_digest,
3977                process_start_time: snapshot.process_start_time,
3978                last_exit: snapshot.last_exit,
3979                health,
3980            },
3981            snapshot.spawned_file_identity.clone(),
3982        ))
3983    }
3984
3985    #[cfg(test)]
3986    pub(crate) fn hold_snapshot_for_test(
3987        &self,
3988        acquired: std::sync::mpsc::Sender<()>,
3989        hold: Duration,
3990    ) -> std::thread::JoinHandle<()> {
3991        let snapshot = Arc::clone(&self.inner.snapshot);
3992        std::thread::spawn(move || {
3993            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3994            acquired
3995                .send(())
3996                .expect("test receiver waits for snapshot lock");
3997            std::thread::sleep(hold);
3998        })
3999    }
4000
4001    /// The status and the running-image check for `supervisor.provenance`,
4002    /// taken from one status read. The exec acknowledgement can land between
4003    /// two separate reads, and the reply would then pair "no pid yet" with an
4004    /// image observed after the module started, which describes no single
4005    /// moment.
4006    pub(crate) async fn status_and_running_image_agreement(
4007        &self,
4008    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
4009        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
4010        let image = self
4011            .inner
4012            .provenance_probe
4013            .observe(
4014                status.pid,
4015                status.spawned_from.as_deref(),
4016                identity,
4017                status.process_start_time,
4018            )
4019            .await;
4020        Ok((status, image))
4021    }
4022
4023    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4024        let snapshot = match lock_snapshot(&self.inner.snapshot) {
4025            Ok(snapshot) => snapshot.clone(),
4026            Err(_) => {
4027                return subc_control::RunningImageAgreement::Unavailable {
4028                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
4029                };
4030            }
4031        };
4032        self.inner
4033            .provenance_probe
4034            .observe(
4035                snapshot.reported_pid(),
4036                snapshot.spawned_from.as_deref(),
4037                snapshot.spawned_file_identity.clone(),
4038                snapshot.process_start_time,
4039            )
4040            .await
4041    }
4042
4043    /// Memory and CPU time of the module's current process, read now. Only the
4044    /// process the supervisor spawned is read, not processes it has started.
4045    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4046        #[cfg(windows)]
4047        {
4048            let image = lock_snapshot(&self.inner.snapshot)
4049                .ok()
4050                .and_then(|snapshot| {
4051                    snapshot
4052                        .spawned_file_identity
4053                        .as_ref()
4054                        .and_then(|identity| identity.windows_image.clone())
4055                });
4056            match image {
4057                Some(image) => {
4058                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
4059                }
4060                None => subc_control::ChildResourceUsage::Unavailable {
4061                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4062                },
4063            }
4064        }
4065        #[cfg(not(windows))]
4066        {
4067            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4068                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4069                Err(_) => {
4070                    return subc_control::ChildResourceUsage::Unavailable {
4071                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4072                    }
4073                }
4074            };
4075            crate::child_resources::read(pid, start_time)
4076        }
4077    }
4078
4079    #[cfg(all(test, any(target_os = "linux", windows)))]
4080    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4081        self.inner
4082            .snapshot
4083            .try_lock()
4084            .map(|snapshot| snapshot.subtree_target.clone())
4085            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4086    }
4087
4088    #[cfg(test)]
4089    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4090        self.inner.snapshot.lock().unwrap().subtree_target = target;
4091    }
4092
4093    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4094        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4095        Ok(match snapshot.state {
4096            ModuleState::Restarting => true,
4097            ModuleState::Failed | ModuleState::Disabled => false,
4098            _ if snapshot.enabled && snapshot.respawn_pending => true,
4099            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4100        })
4101    }
4102
4103    #[cfg(test)]
4104    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4105        self.is_warming_with_snapshot_lock(None)
4106    }
4107
4108    pub(crate) fn is_warming_for_control(
4109        &self,
4110        caller: &'static str,
4111    ) -> Result<bool, SuperviseError> {
4112        self.is_warming_with_snapshot_lock(Some(caller))
4113    }
4114
4115    fn is_warming_with_snapshot_lock(
4116        &self,
4117        caller: Option<&'static str>,
4118    ) -> Result<bool, SuperviseError> {
4119        let snapshot = match caller {
4120            Some(caller) => {
4121                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4122            }
4123            None => lock_snapshot(&self.inner.snapshot)?,
4124        }
4125        .clone();
4126        Ok(matches!(
4127            snapshot.state,
4128            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4129        ))
4130    }
4131
4132    /// Drain the module and stop monitoring it.
4133    pub async fn drain(&self) -> Result<(), SuperviseError> {
4134        self.stop().await
4135    }
4136
4137    pub(crate) async fn retire_requested(
4138        &self,
4139        requester: Option<Principal>,
4140    ) -> Result<(), SuperviseError> {
4141        match self.state()? {
4142            ModuleState::Stopped => return Ok(()),
4143            ModuleState::Failed
4144                if lock_snapshot(&self.inner.snapshot)?
4145                    .failed_retry
4146                    .pending
4147                    .is_none() =>
4148            {
4149                return Ok(())
4150            }
4151            ModuleState::Failed
4152            | ModuleState::Starting
4153            | ModuleState::Running
4154            | ModuleState::Unresponsive
4155            | ModuleState::Restarting
4156            | ModuleState::Draining
4157            | ModuleState::Disabled => {}
4158        }
4159
4160        let (reply_tx, reply_rx) = oneshot::channel();
4161        self.inner
4162            .commands
4163            .send(SupervisorCommand::Retire {
4164                requester,
4165                reply: reply_tx,
4166            })
4167            .await
4168            .map_err(|_| SuperviseError::CommandClosed {
4169                module_id: self.inner.module_id.clone(),
4170            })?;
4171        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4172            module_id: self.inner.module_id.clone(),
4173        })?
4174    }
4175
4176    pub async fn stop(&self) -> Result<(), SuperviseError> {
4177        self.stop_requested(None).await
4178    }
4179
4180    pub(crate) async fn stop_requested(
4181        &self,
4182        requester: Option<Principal>,
4183    ) -> Result<(), SuperviseError> {
4184        match self.state()? {
4185            ModuleState::Stopped => return Ok(()),
4186            ModuleState::Failed
4187                if lock_snapshot(&self.inner.snapshot)?
4188                    .failed_retry
4189                    .pending
4190                    .is_none() =>
4191            {
4192                return Ok(())
4193            }
4194            ModuleState::Failed
4195            | ModuleState::Starting
4196            | ModuleState::Running
4197            | ModuleState::Unresponsive
4198            | ModuleState::Restarting
4199            | ModuleState::Draining
4200            | ModuleState::Disabled => {}
4201        }
4202
4203        let (reply_tx, reply_rx) = oneshot::channel();
4204        self.inner
4205            .commands
4206            .send(SupervisorCommand::Drain {
4207                requester,
4208                reply: reply_tx,
4209            })
4210            .await
4211            .map_err(|_| SuperviseError::CommandClosed {
4212                module_id: self.inner.module_id.clone(),
4213            })?;
4214        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4215            module_id: self.inner.module_id.clone(),
4216        })?
4217    }
4218
4219    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4220        self.restart_requested(drain_timeout_ms, None).await
4221    }
4222
4223    pub(crate) async fn restart_requested(
4224        &self,
4225        drain_timeout_ms: Option<u64>,
4226        requester: Option<Principal>,
4227    ) -> Result<(), SuperviseError> {
4228        let queued_at = Instant::now();
4229        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4230        let (reply_tx, reply_rx) = oneshot::channel();
4231        self.inner
4232            .commands
4233            .send(SupervisorCommand::Restart {
4234                requester,
4235                drain_timeout_ms,
4236                received_at_generation,
4237                queued_at,
4238                reply: reply_tx,
4239            })
4240            .await
4241            .map_err(|_| SuperviseError::CommandClosed {
4242                module_id: self.inner.module_id.clone(),
4243            })?;
4244        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4245            module_id: self.inner.module_id.clone(),
4246        })?
4247    }
4248
4249    /// Blue/green restart: see `SupervisorCommand::Swap` and the
4250    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4251    /// process then drains in the background of the supervise loop) or has
4252    /// failed, leaving the old process serving.
4253    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4254        self.swap_requested(ready_timeout, None).await
4255    }
4256
4257    pub(crate) async fn swap_requested(
4258        &self,
4259        ready_timeout: Option<Duration>,
4260        requester: Option<Principal>,
4261    ) -> Result<(), SuperviseError> {
4262        let (reply_tx, reply_rx) = oneshot::channel();
4263        self.inner
4264            .commands
4265            .send(SupervisorCommand::Swap {
4266                requester,
4267                ready_timeout,
4268                reply: reply_tx,
4269            })
4270            .await
4271            .map_err(|_| SuperviseError::CommandClosed {
4272                module_id: self.inner.module_id.clone(),
4273            })?;
4274        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4275            module_id: self.inner.module_id.clone(),
4276        })?
4277    }
4278
4279    pub async fn reload(&self) -> Result<(), SuperviseError> {
4280        self.reload_requested(None).await
4281    }
4282
4283    pub(crate) async fn reload_requested(
4284        &self,
4285        requester: Option<Principal>,
4286    ) -> Result<(), SuperviseError> {
4287        let (reply_tx, reply_rx) = oneshot::channel();
4288        self.inner
4289            .commands
4290            .send(SupervisorCommand::Reload {
4291                requester,
4292                reply: reply_tx,
4293            })
4294            .await
4295            .map_err(|_| SuperviseError::CommandClosed {
4296                module_id: self.inner.module_id.clone(),
4297            })?;
4298        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4299            module_id: self.inner.module_id.clone(),
4300        })?
4301    }
4302
4303    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4304        self.set_enabled_requested(enabled, None).await
4305    }
4306
4307    pub(crate) async fn set_enabled_requested(
4308        &self,
4309        enabled: bool,
4310        requester: Option<Principal>,
4311    ) -> Result<bool, SuperviseError> {
4312        let (reply_tx, reply_rx) = oneshot::channel();
4313        self.inner
4314            .commands
4315            .send(SupervisorCommand::SetEnabled {
4316                requester,
4317                enabled,
4318                reply: reply_tx,
4319            })
4320            .await
4321            .map_err(|_| SuperviseError::CommandClosed {
4322                module_id: self.inner.module_id.clone(),
4323            })?;
4324        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4325            module_id: self.inner.module_id.clone(),
4326        })?
4327    }
4328
4329    /// The current process's protocol, or the configured protocol when down.
4330    /// A rescan stores the next launch spec without changing how an existing
4331    /// process registers, serves routes, is probed, or exits.
4332    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4333        let configured = self
4334            .inner
4335            .configuration
4336            .lock()
4337            .map_err(|_| SuperviseError::StatePoisoned {
4338                module_id: Some(self.inner.module_id.clone()),
4339            })?
4340            .spec
4341            .protocol;
4342        let state = lock_snapshot(&self.inner.snapshot)?;
4343        Ok(state.spawned_protocol.unwrap_or(configured))
4344    }
4345
4346    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4347        let configuration =
4348            self.inner
4349                .configuration
4350                .lock()
4351                .map_err(|_| SuperviseError::StatePoisoned {
4352                    module_id: Some(self.inner.module_id.clone()),
4353                })?;
4354        Ok((configuration.spec.clone(), configuration.health.clone()))
4355    }
4356
4357    /// Replace this module's launch spec, keeping its health and drain policy,
4358    /// the way a rescan does for a changed config entry. The running process is
4359    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4360    #[cfg(any(test, feature = "test-support"))]
4361    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4362        let (_, health) = self.configuration()?;
4363        let drain_timeout_ms = u64::try_from(
4364            self.inner
4365                .effective_drain_timeout
4366                .lock()
4367                .unwrap_or_else(|poisoned| poisoned.into_inner())
4368                .as_millis(),
4369        )
4370        .ok();
4371        self.update_configuration(spec, health, drain_timeout_ms)
4372            .await
4373    }
4374
4375    pub(crate) async fn update_configuration(
4376        &self,
4377        spec: ModuleSpec,
4378        health: HealthConfig,
4379        drain_timeout_ms: Option<u64>,
4380    ) -> Result<(), SuperviseError> {
4381        if spec.module_id != self.inner.module_id {
4382            return Err(SuperviseError::InvalidSpec {
4383                reason: "a supervised module's module_id cannot be changed".to_string(),
4384            });
4385        }
4386        validate_spec(&spec)?;
4387        let (reply_tx, reply_rx) = oneshot::channel();
4388        self.inner
4389            .commands
4390            .send(SupervisorCommand::UpdateConfiguration {
4391                spec: spec.clone(),
4392                health: health.clone(),
4393                drain_timeout_ms,
4394                reply: reply_tx,
4395            })
4396            .await
4397            .map_err(|_| SuperviseError::CommandClosed {
4398                module_id: self.inner.module_id.clone(),
4399            })?;
4400        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4401            module_id: self.inner.module_id.clone(),
4402        })?;
4403        let mut configuration =
4404            self.inner
4405                .configuration
4406                .lock()
4407                .map_err(|_| SuperviseError::StatePoisoned {
4408                    module_id: Some(self.inner.module_id.clone()),
4409                })?;
4410        configuration.spec = spec;
4411        configuration.health = health;
4412        Ok(())
4413    }
4414}
4415
4416impl Drop for SupervisedModuleInner {
4417    fn drop(&mut self) {
4418        let Ok(mut monitor) = self.monitor.lock() else {
4419            return;
4420        };
4421        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4422            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4423                state.state = ModuleState::Stopped;
4424                clear_current_process_facts(state);
4425            });
4426            monitor.abort();
4427        }
4428        let _ = monitor.take();
4429    }
4430}
4431
4432#[derive(Debug)]
4433enum SupervisorCommand {
4434    Drain {
4435        requester: Option<Principal>,
4436        reply: oneshot::Sender<Result<(), SuperviseError>>,
4437    },
4438    Retire {
4439        requester: Option<Principal>,
4440        reply: oneshot::Sender<Result<(), SuperviseError>>,
4441    },
4442    Restart {
4443        requester: Option<Principal>,
4444        /// Operator override for this one restart's drain budget, in ms. `None`
4445        /// uses the module's configured/default budget; `Some(0)` cuts
4446        /// immediately (wedge bounce: a stuck request never settles, so
4447        /// waiting only delays recovery).
4448        drain_timeout_ms: Option<u64>,
4449        /// The module's `spawn_generation` when the request was received, before
4450        /// it waited in the command queue. A queued restart whose module has
4451        /// since spawned a newer process is already satisfied (see the handler).
4452        received_at_generation: u64,
4453        /// When the request entered the command queue, so the handler can log
4454        /// how long it waited behind the loop's other work.
4455        queued_at: Instant,
4456        reply: oneshot::Sender<Result<(), SuperviseError>>,
4457    },
4458    Reload {
4459        requester: Option<Principal>,
4460        reply: oneshot::Sender<Result<(), SuperviseError>>,
4461    },
4462    SetEnabled {
4463        requester: Option<Principal>,
4464        enabled: bool,
4465        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4466    },
4467    UpdateConfiguration {
4468        spec: ModuleSpec,
4469        health: HealthConfig,
4470        /// Per-module drain override from the new config; `None` re-resolves to
4471        /// the supervisor-wide default.
4472        drain_timeout_ms: Option<u64>,
4473        reply: oneshot::Sender<()>,
4474    },
4475    Swap {
4476        requester: Option<Principal>,
4477        /// How long the candidate may take to register and declare itself
4478        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4479        ready_timeout: Option<Duration>,
4480        /// Answered at cutover or failure; the incumbent's drain follows.
4481        reply: oneshot::Sender<Result<(), SuperviseError>>,
4482    },
4483}
4484
4485#[derive(Debug)]
4486pub enum SuperviseError {
4487    InvalidSpec {
4488        reason: String,
4489    },
4490    Spawn {
4491        program: PathBuf,
4492        source: io::Error,
4493        cgroup_path: Option<PathBuf>,
4494    },
4495    Cgroup {
4496        module_id: String,
4497        source: io::Error,
4498    },
4499    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4500    /// than spawn a reserved module without its identity binding.
4501    LaunchNonce {
4502        reason: String,
4503    },
4504    Wait {
4505        module_id: String,
4506        source: io::Error,
4507    },
4508    Kill {
4509        module_id: String,
4510        source: io::Error,
4511    },
4512    Forwarding(ForwardingError),
4513    Registry(RegistryError),
4514    ReloadUnavailable {
4515        module_id: String,
4516        reason: String,
4517    },
4518    /// An operator restart/reload was requested for a module that is currently
4519    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4520    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4521    /// by a restart, so these commands are rejected instead of re-enabling it.
4522    Disabled {
4523        module_id: String,
4524    },
4525    ReloadFailed {
4526        module_id: String,
4527        reason: String,
4528    },
4529    RegistrationStillActive {
4530        module_id: String,
4531        waited: Duration,
4532    },
4533    StatePoisoned {
4534        module_id: Option<String>,
4535    },
4536    CommandClosed {
4537        module_id: String,
4538    },
4539    /// A restart or reload arrived while a swap's candidate was warming. The
4540    /// swap owns the module until it cuts over or fails; a stop or disable
4541    /// would have aborted it instead.
4542    SwapInProgress {
4543        module_id: String,
4544    },
4545    /// A swap was refused before anything was spawned.
4546    SwapRefused {
4547        module_id: String,
4548        reason: SwapRefusal,
4549    },
4550    /// A swap spawned a candidate and gave up on it. The candidate has been
4551    /// killed and its slot freed; the incumbent was left serving and was never
4552    /// drained, except in the one `CutoverLost` case described on that arm.
4553    SwapFailed {
4554        module_id: String,
4555        arm: SwapFailureArm,
4556        detail: String,
4557        /// How the candidate exited, when it exited on its own before the
4558        /// supervisor gave up on it.
4559        candidate_exit: Option<ExitReport>,
4560    },
4561}
4562
4563/// Why a swap was refused before a candidate was spawned.
4564#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4565pub enum SwapRefusal {
4566    /// The module's config does not declare `overlap: "safe"`.
4567    OverlapExclusive,
4568    /// The module is not registered, so there is no incumbent to keep serving
4569    /// and nothing a swap would improve on; a plain restart is the tool.
4570    NotRegistered,
4571    /// The module does not speak the subc wire, so a candidate could never
4572    /// register or declare itself ready.
4573    ProtocolNone,
4574    /// The supervisor lacks the forwarding table (to cut routes over) or the
4575    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4576    NotConfigured,
4577    /// A swap is already open for this module.
4578    AlreadySwapping,
4579}
4580
4581impl SwapRefusal {
4582    pub fn as_str(self) -> &'static str {
4583        match self {
4584            Self::OverlapExclusive => "overlap_exclusive",
4585            Self::NotRegistered => "not_registered",
4586            Self::ProtocolNone => "protocol_none",
4587            Self::NotConfigured => "not_configured",
4588            Self::AlreadySwapping => "already_swapping",
4589        }
4590    }
4591}
4592
4593/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4594/// serving and undrained; see `CutoverLost`.
4595#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4596pub enum SwapFailureArm {
4597    /// The candidate process could not be started.
4598    SpawnFailed,
4599    /// The candidate did not register within the readiness budget.
4600    NeverRegistered,
4601    /// The candidate registered but did not declare itself ready in time.
4602    NeverReady,
4603    /// The candidate exited before cutover.
4604    CandidateExited,
4605    /// The candidate declared itself ready but failed its health probe.
4606    CandidateUnhealthy,
4607    /// An operator stop, disable or retire arrived while the candidate warmed.
4608    /// The candidate was killed and the operator's command then carried out on
4609    /// the incumbent.
4610    Interrupted,
4611    /// The candidate's connection closed at the moment of cutover. If it
4612    /// closed before forwarding moved, the incumbent is untouched. If it closed
4613    /// between the forwarding and registry halves of cutover, forwarding can no
4614    /// longer route to the incumbent, so the module is restarted plainly.
4615    CutoverLost,
4616}
4617
4618impl SwapFailureArm {
4619    pub fn as_str(self) -> &'static str {
4620        match self {
4621            Self::SpawnFailed => "spawn_failed",
4622            Self::NeverRegistered => "never_registered",
4623            Self::NeverReady => "never_ready",
4624            Self::CandidateExited => "candidate_exited",
4625            Self::CandidateUnhealthy => "candidate_unhealthy",
4626            Self::Interrupted => "interrupted",
4627            Self::CutoverLost => "cutover_lost",
4628        }
4629    }
4630}
4631
4632impl fmt::Display for SuperviseError {
4633    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4634        match self {
4635            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4636            Self::Spawn {
4637                program,
4638                source,
4639                cgroup_path: Some(cgroup_path),
4640            } => write!(
4641                f,
4642                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4643                cgroup_path.display(),
4644                program.display()
4645            ),
4646            Self::Spawn {
4647                program,
4648                source,
4649                cgroup_path: None,
4650            } => write!(
4651                f,
4652                "failed to spawn module '{}': {source}",
4653                program.display()
4654            ),
4655            Self::Cgroup { module_id, source } => {
4656                write!(
4657                    f,
4658                    "failed to prepare cgroup for module '{module_id}': {source}"
4659                )
4660            }
4661            Self::LaunchNonce { reason } => {
4662                write!(
4663                    f,
4664                    "failed to generate reserved-module launch nonce: {reason}"
4665                )
4666            }
4667            Self::Wait { module_id, source } => {
4668                write!(f, "failed to wait for module '{module_id}': {source}")
4669            }
4670            Self::Kill { module_id, source } => {
4671                write!(f, "failed to kill module '{module_id}': {source}")
4672            }
4673            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4674            Self::Registry(err) => write!(f, "registry error: {err}"),
4675            Self::ReloadUnavailable { module_id, reason } => {
4676                write!(f, "reload unavailable for module '{module_id}': {reason}")
4677            }
4678            Self::Disabled { module_id } => {
4679                write!(
4680                    f,
4681                    "module '{module_id}' is disabled; enable it before restart or reload"
4682                )
4683            }
4684            Self::ReloadFailed { module_id, reason } => {
4685                write!(f, "reload failed for module '{module_id}': {reason}")
4686            }
4687            Self::RegistrationStillActive { module_id, waited } => write!(
4688                f,
4689                "module '{module_id}' registration remained active after waiting {waited:?}"
4690            ),
4691            Self::StatePoisoned { module_id } => match module_id {
4692                Some(module_id) => {
4693                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4694                }
4695                None => write!(f, "supervisor state was poisoned"),
4696            },
4697            Self::CommandClosed { module_id } => {
4698                write!(
4699                    f,
4700                    "supervisor command channel for module '{module_id}' is closed"
4701                )
4702            }
4703            Self::SwapInProgress { module_id } => write!(
4704                f,
4705                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4706            ),
4707            Self::SwapRefused { module_id, reason } => match reason {
4708                SwapRefusal::OverlapExclusive => write!(
4709                    f,
4710                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4711                ),
4712                SwapRefusal::NotRegistered => write!(
4713                    f,
4714                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4715                ),
4716                SwapRefusal::ProtocolNone => write!(
4717                    f,
4718                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4719                ),
4720                SwapRefusal::NotConfigured => write!(
4721                    f,
4722                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4723                ),
4724                SwapRefusal::AlreadySwapping => {
4725                    write!(f, "module '{module_id}' is already being swapped")
4726                }
4727            },
4728            Self::SwapFailed {
4729                module_id,
4730                arm,
4731                detail,
4732                ..
4733            } => write!(
4734                f,
4735                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4736                arm.as_str()
4737            ),
4738        }
4739    }
4740}
4741
4742impl Error for SuperviseError {
4743    fn source(&self) -> Option<&(dyn Error + 'static)> {
4744        match self {
4745            Self::Spawn { source, .. }
4746            | Self::Cgroup { source, .. }
4747            | Self::Wait { source, .. }
4748            | Self::Kill { source, .. } => Some(source),
4749            Self::Forwarding(err) => Some(err),
4750            Self::Registry(err) => Some(err),
4751            Self::LaunchNonce { .. }
4752            | Self::InvalidSpec { .. }
4753            | Self::ReloadUnavailable { .. }
4754            | Self::Disabled { .. }
4755            | Self::ReloadFailed { .. }
4756            | Self::RegistrationStillActive { .. }
4757            | Self::StatePoisoned { .. }
4758            | Self::CommandClosed { .. }
4759            | Self::SwapInProgress { .. }
4760            | Self::SwapRefused { .. }
4761            | Self::SwapFailed { .. } => None,
4762        }
4763    }
4764}
4765
4766pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4767    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4768        return Err(SuperviseError::InvalidSpec {
4769            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4770        });
4771    }
4772    if spec.module_id.trim().is_empty() {
4773        return Err(SuperviseError::InvalidSpec {
4774            reason: "module_id must not be empty".to_string(),
4775        });
4776    }
4777
4778    Ok(())
4779}
4780
4781#[derive(Debug, Default)]
4782struct HealthProbeRuntime {
4783    spawn_generation: Option<u64>,
4784    configured_health: Option<HealthConfig>,
4785    registered_connection: Option<crate::ConnectionId>,
4786    advertised: bool,
4787    next_probe_at: Option<Instant>,
4788    probe_index: u64,
4789}
4790
4791fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4792    lock_snapshot(snapshot)
4793        .ok()
4794        .and_then(|state| state.spawned_protocol)
4795        .unwrap_or(spec.protocol)
4796}
4797
4798impl HealthProbeRuntime {
4799    fn refresh_registration(
4800        &mut self,
4801        spec: &ModuleSpec,
4802        runtime: &SupervisorRuntimeConfig,
4803        registry: &Registry,
4804        snapshot: &SharedSnapshot,
4805    ) {
4806        let generation = lock_snapshot(snapshot)
4807            .ok()
4808            .map(|state| state.spawn_generation);
4809        if self.spawn_generation != generation {
4810            // A replacement cannot reuse a deadline that is already due for its
4811            // predecessor. HTTP children get a full cadence to start; wire
4812            // children still wait for their own health-advertising HELLO.
4813            *self = Self {
4814                spawn_generation: generation,
4815                ..Self::default()
4816            };
4817        }
4818        if self.configured_health.as_ref() != Some(&runtime.health) {
4819            self.configured_health = Some(runtime.health.clone());
4820            self.next_probe_at = None;
4821            self.registered_connection = None;
4822            self.probe_index = 0;
4823        }
4824        // A non-wire process never registers. Only an explicitly configured
4825        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4826        // health failure for that kind of process.
4827        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4828            self.registered_connection = None;
4829            self.advertised = runtime.health.http.is_some();
4830            if !self.advertised {
4831                self.next_probe_at = None;
4832                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4833                    let unknown = ModuleHealthStatus::default();
4834                    if state.health != unknown {
4835                        state.health = unknown;
4836                        state.last_probe_evidence = None;
4837                    }
4838                });
4839            } else if self.next_probe_at.is_none() {
4840                self.next_probe_at = Some(
4841                    Instant::now()
4842                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4843                );
4844            }
4845            return;
4846        }
4847
4848        let registration = match registry.get_module(&spec.module_id) {
4849            Ok(registration) => registration,
4850            Err(err) => {
4851                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4852                self.advertised = false;
4853                self.next_probe_at = None;
4854                return;
4855            }
4856        };
4857
4858        let Some(registration) = registration else {
4859            self.registered_connection = None;
4860            self.advertised = false;
4861            self.next_probe_at = None;
4862            return;
4863        };
4864
4865        let advertised = registration
4866            .control_ops
4867            .iter()
4868            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4869        if !advertised {
4870            self.registered_connection = Some(registration.connection_id);
4871            self.advertised = false;
4872            self.next_probe_at = None;
4873            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4874                if state.health.status != SupervisorHealthStatus::Unknown
4875                    || state.health.consecutive_failures != 0
4876                    || state.health.last_probe_ms.is_some()
4877                    || state.health.detail.is_some()
4878                    || state.health.metrics.is_some()
4879                {
4880                    state.health.status = SupervisorHealthStatus::Unknown;
4881                    state.last_probe_evidence = None;
4882                    state.health.consecutive_failures = 0;
4883                    state.health.last_probe_ms = None;
4884                    state.health.detail = None;
4885                    state.health.metrics = None;
4886                }
4887            });
4888            return;
4889        }
4890
4891        let reregistered = self.registered_connection != Some(registration.connection_id);
4892        self.registered_connection = Some(registration.connection_id);
4893        self.advertised = true;
4894        if reregistered || self.next_probe_at.is_none() {
4895            self.probe_index = 0;
4896            self.next_probe_at = Some(
4897                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4898            );
4899            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4900                state.health.status = SupervisorHealthStatus::Unknown;
4901                state.last_probe_evidence = None;
4902                state.health.consecutive_failures = 0;
4903                state.health.detail = None;
4904                state.health.metrics = None;
4905            });
4906        }
4907    }
4908
4909    fn wake_at(&self) -> Option<Instant> {
4910        if !self.advertised {
4911            return None;
4912        }
4913        self.next_probe_at
4914    }
4915
4916    fn due(&self) -> bool {
4917        self.advertised
4918            && self
4919                .next_probe_at
4920                .is_some_and(|next| Instant::now() >= next)
4921    }
4922
4923    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4924        self.probe_index = self.probe_index.wrapping_add(1);
4925        self.next_probe_at = Some(
4926            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4927        );
4928    }
4929}
4930
4931/// Evidence is separate from diagnostic prose so host starvation can hold only
4932/// silence-based restarts. A missing lane is still actionable during overload;
4933/// a timeout can instead mean the host has not scheduled a healthy module.
4934#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4935enum HealthProbeEvidence {
4936    /// The module's control lane is gone. Proof of death.
4937    LaneDead,
4938    /// No reply within the deadline. Proves nothing about the module's state.
4939    NoAnswer,
4940    /// The module replied, but not with a usable health report. Proves it is alive.
4941    BadAnswer,
4942    /// The daemon could not ask. Says nothing about the module.
4943    Misconfigured,
4944}
4945
4946#[derive(Debug)]
4947struct HealthProbeError {
4948    evidence: HealthProbeEvidence,
4949    message: String,
4950}
4951
4952impl HealthProbeError {
4953    fn lane_dead(message: impl Into<String>) -> Self {
4954        Self::with(HealthProbeEvidence::LaneDead, message)
4955    }
4956
4957    fn no_answer(message: impl Into<String>) -> Self {
4958        Self::with(HealthProbeEvidence::NoAnswer, message)
4959    }
4960
4961    fn bad_answer(message: impl Into<String>) -> Self {
4962        Self::with(HealthProbeEvidence::BadAnswer, message)
4963    }
4964
4965    fn misconfigured(message: impl Into<String>) -> Self {
4966        Self::with(HealthProbeEvidence::Misconfigured, message)
4967    }
4968
4969    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4970        Self {
4971            evidence,
4972            message: message.into(),
4973        }
4974    }
4975
4976    /// Whether this observation is proof the module cannot serve.
4977    ///
4978    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4979    /// variant that fires under CPU starvation, and treating it as proof is the
4980    /// defect this enum exists to make impossible to reintroduce silently.
4981    ///
4982    /// Outside a host-overload hold, silence still reaches the normal failure
4983    /// threshold so an isolated, genuinely wedged module can recover.
4984    #[allow(dead_code)]
4985    fn is_proof_of_death(&self) -> bool {
4986        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4987    }
4988
4989    /// Short stable label for logs and the health snapshot.
4990    ///
4991    /// An operator reading `ck health` currently cannot tell "the module is gone"
4992    /// from "the module did not answer in five seconds", because both render as
4993    /// prose in the same field. These labels are what make the two
4994    /// distinguishable at a glance, and they are what a later restart-policy
4995    /// change will be argued from.
4996    fn label(&self) -> &'static str {
4997        match self.evidence {
4998            HealthProbeEvidence::LaneDead => "lane-dead",
4999            HealthProbeEvidence::NoAnswer => "no-answer",
5000            HealthProbeEvidence::BadAnswer => "bad-answer",
5001            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5002        }
5003    }
5004}
5005
5006impl fmt::Display for HealthProbeError {
5007    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5008        f.write_str(&self.message)
5009    }
5010}
5011
5012async fn run_health_probe_cycle(
5013    spec: &ModuleSpec,
5014    runtime: &SupervisorRuntimeConfig,
5015    registry: &Registry,
5016    process_liveness: &SupervisorProcessLiveness,
5017    snapshot: &SharedSnapshot,
5018    child: &mut Option<SupervisedChild>,
5019) {
5020    let now_ms = unix_ms_now();
5021    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5022        .then_some(runtime.health.http.as_deref())
5023        .flatten();
5024    let result = match http {
5025        Some(url) => probe_http_health(url, runtime.health.deadline).await,
5026        None => probe_module_health(&spec.module_id, runtime, None).await,
5027    };
5028    match result {
5029        Ok(report) => {
5030            handle_health_report(
5031                spec,
5032                runtime,
5033                registry,
5034                process_liveness,
5035                snapshot,
5036                child,
5037                report,
5038                now_ms,
5039            )
5040            .await;
5041        }
5042        Err(err) => {
5043            if http.is_some() {
5044                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5045                    state.health.status = SupervisorHealthStatus::Failing;
5046                });
5047            }
5048            handle_health_probe_failure(
5049                spec,
5050                runtime,
5051                registry,
5052                process_liveness,
5053                snapshot,
5054                child,
5055                err,
5056                now_ms,
5057            )
5058            .await;
5059        }
5060    }
5061}
5062
5063pub(crate) struct HttpProbeTarget<'a> {
5064    address: std::net::SocketAddr,
5065    localhost: bool,
5066    authority: &'a str,
5067    path: String,
5068}
5069
5070/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
5071/// or TLS. A URL cannot turn a local health check into an outbound connection.
5072pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5073    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5074        return Err("must not contain whitespace, controls, or a fragment".into());
5075    }
5076    let rest = url
5077        .strip_prefix("http://")
5078        .ok_or("must use plain http://")?;
5079    let split = rest.find(['/', '?']).unwrap_or(rest.len());
5080    let (authority, suffix) = rest.split_at(split);
5081    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5082        ("::1", rest)
5083    } else {
5084        let split = authority.find(':').unwrap_or(authority.len());
5085        authority.split_at(split)
5086    };
5087    let ip: std::net::IpAddr = match host {
5088        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5089        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5090        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5091    };
5092    let port = if port.is_empty() {
5093        80
5094    } else {
5095        port.strip_prefix(':')
5096            .and_then(|p| p.parse::<u16>().ok())
5097            .filter(|p| *p > 0)
5098            .ok_or("must have a valid nonzero TCP port")?
5099    };
5100    let path = if suffix.is_empty() {
5101        "/".into()
5102    } else if suffix.starts_with('?') {
5103        format!("/{suffix}")
5104    } else {
5105        suffix.into()
5106    };
5107    Ok(HttpProbeTarget {
5108        address: std::net::SocketAddr::new(ip, port),
5109        localhost: host == "localhost",
5110        authority,
5111        path,
5112    })
5113}
5114
5115async fn probe_http_health(
5116    url: &str,
5117    deadline: Duration,
5118) -> Result<HealthReport, HealthProbeError> {
5119    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5120    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5121    // Keep partial diagnostics outside the timed future so cancellation does
5122    // not discard a status line or body bytes already received.
5123    let mut response_status = String::new();
5124    let mut body = Vec::new();
5125    let probe = async {
5126        // Resolve localhost ourselves so a hosts-file override cannot turn
5127        // this into an outbound request, while IPv6-only local servers work.
5128        let connection = match tokio::net::TcpStream::connect(target.address).await {
5129            Err(_) if target.localhost => {
5130                tokio::net::TcpStream::connect((
5131                    std::net::Ipv6Addr::LOCALHOST,
5132                    target.address.port(),
5133                ))
5134                .await
5135            }
5136            result => result,
5137        };
5138        let mut stream = connection.map_err(|error| {
5139            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5140        })?;
5141        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5142            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5143        let mut reader = BufReader::new(stream);
5144        let mut budget = 16 * 1024;
5145        let status = http_line(&mut reader, &mut budget).await?;
5146        let mut words = status.split_ascii_whitespace();
5147        let version = words.next();
5148        let code = words
5149            .next()
5150            .filter(|word| word.len() == 3)
5151            .and_then(|word| word.parse::<u16>().ok());
5152        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5153            || !code.is_some_and(|code| (100..600).contains(&code))
5154        {
5155            return Err(HealthProbeError::bad_answer(format!(
5156                "invalid HTTP status: {status}"
5157            )));
5158        }
5159        let code = code.expect("validated status code");
5160        response_status = status.clone();
5161        let mut length = None;
5162        let mut chunked = false;
5163        loop {
5164            let line = http_line(&mut reader, &mut budget).await?;
5165            if line.is_empty() {
5166                break;
5167            }
5168            if let Some((name, value)) = line.split_once(':') {
5169                if name.eq_ignore_ascii_case("content-length") {
5170                    length = Some(value.trim().parse::<u64>().map_err(|_| {
5171                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
5172                    })?);
5173                } else if name.eq_ignore_ascii_case("transfer-encoding") {
5174                    chunked = value.trim().eq_ignore_ascii_case("chunked");
5175                }
5176            }
5177        }
5178        if chunked {
5179            while body.len() < 200 {
5180                let line = http_line(&mut reader, &mut budget).await?;
5181                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5182                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5183                if size == 0 {
5184                    break;
5185                }
5186                let count = size.min((200 - body.len()) as u64) as usize;
5187                let start = body.len();
5188                (&mut reader)
5189                    .take(count as u64)
5190                    .read_to_end(&mut body)
5191                    .await
5192                    .map_err(|error| {
5193                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5194                    })?;
5195                if body.len() - start != count {
5196                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5197                }
5198                if size > count as u64 || body.len() == 200 {
5199                    break;
5200                }
5201                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5202                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5203                }
5204            }
5205        } else {
5206            reader
5207                .take(length.unwrap_or(200).min(200))
5208                .read_to_end(&mut body)
5209                .await
5210                .map_err(|error| {
5211                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5212                })?;
5213        }
5214        if (200..300).contains(&code) {
5215            Ok(HealthReport::ok())
5216        } else {
5217            Err(HealthProbeError::bad_answer(
5218                "HTTP health endpoint returned non-2xx",
5219            ))
5220        }
5221    };
5222    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5223        Err(HealthProbeError::no_answer(format!(
5224            "HTTP probe timed out after {deadline:?}"
5225        )))
5226    });
5227    if let Err(error) = &mut result {
5228        if !response_status.is_empty() {
5229            error.message = format!(
5230                "{}; {response_status}: {}",
5231                error.message,
5232                String::from_utf8_lossy(&body)
5233            );
5234        }
5235    }
5236    result
5237}
5238
5239async fn http_line(
5240    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5241    remaining: &mut usize,
5242) -> Result<String, HealthProbeError> {
5243    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5244    let mut line = Vec::new();
5245    (&mut *reader)
5246        .take(*remaining as u64)
5247        .read_until(b'\n', &mut line)
5248        .await
5249        .map_err(|error| {
5250            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5251        })?;
5252    *remaining -= line.len();
5253    if !line.ends_with(b"\r\n") {
5254        return Err(HealthProbeError::bad_answer(
5255            "HTTP headers are incomplete or exceed 16 KiB",
5256        ));
5257    }
5258    line.truncate(line.len() - 2);
5259    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5260}
5261
5262async fn probe_module_health(
5263    module_id: &str,
5264    runtime: &SupervisorRuntimeConfig,
5265    drain_deadline: Option<Instant>,
5266) -> Result<HealthReport, HealthProbeError> {
5267    let Some(forwarding) = runtime.forwarding.as_ref() else {
5268        return Err(HealthProbeError::misconfigured(
5269            "supervisor was not configured with a forwarding table",
5270        ));
5271    };
5272    let probe_started_at = Instant::now();
5273    let mut deadline = probe_started_at + runtime.health.deadline;
5274    if let Some(drain_deadline) = drain_deadline {
5275        deadline = deadline.min(drain_deadline);
5276    }
5277    let pending = if drain_deadline.is_some() {
5278        forwarding.begin_drain_health_probe_rpc_for(
5279            module_id,
5280            MODULE_CONTROL_OP_HEALTH_CHECK,
5281            probe_started_at,
5282            deadline,
5283        )
5284    } else {
5285        forwarding.begin_health_probe_rpc_for(
5286            module_id,
5287            MODULE_CONTROL_OP_HEALTH_CHECK,
5288            probe_started_at,
5289            deadline,
5290        )
5291    }
5292    .map_err(|err| {
5293        // The endpoint is not registered, so there is no live control lane to
5294        // ask. That is the module being absent, not slow.
5295        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5296    })?;
5297    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5298}
5299
5300/// [`probe_module_health`] for one endpoint rather than the id's active one.
5301///
5302/// A swap probes two processes that no by-id lookup reaches: its candidate
5303/// before cutover, and its superseded incumbent (for busy gauges) while the
5304/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5305/// bounds the by-id drain probe.
5306async fn probe_endpoint_health(
5307    endpoint: crate::ModuleEndpointId,
5308    runtime: &SupervisorRuntimeConfig,
5309    deadline_cap: Option<Instant>,
5310) -> Result<HealthReport, HealthProbeError> {
5311    let Some(forwarding) = runtime.forwarding.as_ref() else {
5312        return Err(HealthProbeError::misconfigured(
5313            "supervisor was not configured with a forwarding table",
5314        ));
5315    };
5316    let probe_started_at = Instant::now();
5317    let mut deadline = probe_started_at + runtime.health.deadline;
5318    if let Some(cap) = deadline_cap {
5319        deadline = deadline.min(cap);
5320    }
5321    let pending = forwarding
5322        .begin_endpoint_health_probe_rpc_for(
5323            endpoint,
5324            MODULE_CONTROL_OP_HEALTH_CHECK,
5325            probe_started_at,
5326            deadline,
5327        )
5328        .map_err(|err| {
5329            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5330        })?;
5331    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5332}
5333
5334/// Send a begun health probe and classify its answer.
5335async fn await_health_probe(
5336    forwarding: &ForwardingTable,
5337    pending: PendingModuleControlRpc,
5338    deadline: Instant,
5339    probe_budget: Duration,
5340) -> Result<HealthReport, HealthProbeError> {
5341    let PendingModuleControlRpc {
5342        endpoint,
5343        module_sink,
5344        negotiated_ver,
5345        corr,
5346        receiver,
5347    } = pending;
5348    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5349        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5350    })?;
5351    let frame = Frame::build_with_version(
5352        negotiated_ver,
5353        FrameType::Request,
5354        control_flags(),
5355        0,
5356        0,
5357        corr,
5358        body,
5359    )
5360    .map_err(|err| {
5361        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5362    })?;
5363
5364    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5365    // blocks waiting for capacity when the module's egress queue is full, and an
5366    // unbounded await here freezes the whole supervision actor (it stops polling
5367    // Child::wait and supervisor commands), making the module unrecoverable
5368    // in-band. On timeout the probe fails like any transport failure.
5369    match timeout_at(deadline, module_sink.send(frame)).await {
5370        Ok(Ok(())) => {}
5371        Ok(Err(err)) => {
5372            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5373            // A closed sink means the module's egress channel is gone -- the
5374            // receiving half is dropped when its connection tears down. Proof.
5375            return Err(HealthProbeError::lane_dead(format!(
5376                "failed to send health.check: {err}"
5377            )));
5378        }
5379        Err(_elapsed) => {
5380            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5381            // A full egress queue means the module is not draining its socket, which
5382            // is consistent with a wedged module AND with one whose reader is merely
5383            // starved. Silence, not proof.
5384            return Err(HealthProbeError::no_answer(
5385                "health.check send timed out before enqueue (module egress full)",
5386            ));
5387        }
5388    }
5389
5390    match timeout_at(deadline, receiver).await {
5391        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5392        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5393        // and those prove it is alive even though the probe failed.
5394        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5395            response.health_report().ok_or_else(|| {
5396                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5397            })
5398        }
5399        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5400            format!("health.check rejected: {}", body.message),
5401        )),
5402        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5403            Err(HealthProbeError::lane_dead(message))
5404        }
5405        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5406            Err(HealthProbeError::bad_answer(message))
5407        }
5408        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5409            Err(HealthProbeError::bad_answer(format!(
5410                "expected module-control op '{expected}', got '{actual}'"
5411            )))
5412        }
5413        // A reply that crosses the deadline before this waiter observes it is
5414        // still proof of life. The forwarding path records its end-to-end latency
5415        // before delivering this classification.
5416        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5417            "module answered health.check after its daemon deadline",
5418        )),
5419        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5420            "health.check waiter was canceled before the module responded",
5421        )),
5422        Err(_) => {
5423            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5424            Err(HealthProbeError::no_answer(format!(
5425                "module did not answer health.check within {probe_budget:?}"
5426            )))
5427        }
5428    }
5429}
5430
5431#[allow(clippy::too_many_arguments)]
5432async fn handle_health_report(
5433    spec: &ModuleSpec,
5434    runtime: &SupervisorRuntimeConfig,
5435    registry: &Registry,
5436    process_liveness: &SupervisorProcessLiveness,
5437    snapshot: &SharedSnapshot,
5438    child: &mut Option<SupervisedChild>,
5439    report: HealthReport,
5440    now_ms: u64,
5441) {
5442    let status = supervisor_health_status(report.status);
5443    let detail = report.detail.clone();
5444    let metrics = truncate_health_metrics(report.metrics);
5445    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5446        state.health.status = status;
5447        state.last_probe_evidence = None;
5448        state.health.last_probe_ms = Some(now_ms);
5449        state.health.detail = detail.clone();
5450        state.health.metrics = metrics.clone();
5451        state.health.consecutive_failures = 0;
5452        state.failed_retry.observe_health(
5453            report.status == HealthStatus::Ok,
5454            runtime.restart_policy.window,
5455        );
5456    });
5457
5458    let action = match report.status {
5459        HealthStatus::Ok => return,
5460        HealthStatus::Degraded => runtime.health.on_degraded,
5461        HealthStatus::Failing => runtime.health.on_failing,
5462    };
5463    apply_l3_health_action(
5464        spec,
5465        runtime,
5466        registry,
5467        process_liveness,
5468        snapshot,
5469        child,
5470        status,
5471        detail.as_deref(),
5472        action,
5473        now_ms,
5474    )
5475    .await;
5476}
5477
5478#[allow(clippy::too_many_arguments)]
5479async fn handle_health_probe_failure(
5480    spec: &ModuleSpec,
5481    runtime: &SupervisorRuntimeConfig,
5482    registry: &Registry,
5483    process_liveness: &SupervisorProcessLiveness,
5484    snapshot: &SharedSnapshot,
5485    child: &mut Option<SupervisedChild>,
5486    err: HealthProbeError,
5487    now_ms: u64,
5488) {
5489    let threshold = runtime.health.failure_threshold.max(1);
5490    let held = if err.evidence == HealthProbeEvidence::NoAnswer {
5491        runtime.host_health.record_late_module(&spec.module_id)
5492    } else {
5493        false
5494    };
5495    let mut failures = 0;
5496    // Carry the evidence class into the operator-visible detail. Without it,
5497    // "module did not answer within 5s" and "the control lane is gone" are two
5498    // prose strings in the same field, and the reader has to know the codebase to
5499    // tell which one is proof of anything.
5500    let detail = if held {
5501        format!("[{}] {err}; host overloaded", err.label())
5502    } else {
5503        format!("[{}] {err}", err.label())
5504    };
5505    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506        state.last_probe_evidence = Some(err.evidence);
5507        state
5508            .failed_retry
5509            .observe_health(false, runtime.restart_policy.window);
5510        // A failed wire probe invalidates the last report, even before the
5511        // restart threshold. HTTP probes already mark failures as Failing.
5512        if held {
5513            state.health.status = SupervisorHealthStatus::Degraded;
5514        } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5515            state.health.status = SupervisorHealthStatus::Unknown;
5516        }
5517        state.health.last_probe_ms = Some(now_ms);
5518        // Held silence breaks the streak: after the host recovers, only new
5519        // failures count. Replaying pre-overload misses would restart immediately.
5520        state.health.consecutive_failures = if held {
5521            0
5522        } else {
5523            state.health.consecutive_failures.saturating_add(1)
5524        };
5525        state.health.detail = Some(detail.clone());
5526        state.health.metrics = None;
5527        failures = state.health.consecutive_failures;
5528    });
5529
5530    if held || failures < threshold {
5531        warn!(
5532            module_id = %spec.module_id,
5533            consecutive_failures = failures,
5534            threshold,
5535            evidence = err.label(),
5536            detail = %detail,
5537            "health.check probe failed"
5538        );
5539        return;
5540    }
5541
5542    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5543        state.state = ModuleState::Unresponsive;
5544        state.health.status = SupervisorHealthStatus::Unresponsive;
5545    });
5546    // The evidence class is logged at the kill site because this is the line an
5547    // operator reads after an unexplained restart. A streak of `no-answer` under
5548    // machine load is the known false-positive shape; a `lane-dead` is not.
5549    if runtime.health.critical {
5550        error!(
5551            module_id = %spec.module_id,
5552            status = "unresponsive",
5553            evidence = err.label(),
5554            detail = %detail,
5555            "critical module health alert"
5556        );
5557    } else {
5558        warn!(
5559            module_id = %spec.module_id,
5560            status = "unresponsive",
5561            evidence = err.label(),
5562            detail = %detail,
5563            "module health threshold breached"
5564        );
5565    }
5566    if let Err(err) = health_restart_child(
5567        spec,
5568        runtime,
5569        registry,
5570        process_liveness,
5571        snapshot,
5572        child,
5573        SupervisorHealthStatus::Unresponsive,
5574        Some(&detail),
5575        now_ms,
5576    )
5577    .await
5578    {
5579        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5580    }
5581}
5582
5583#[allow(clippy::too_many_arguments)]
5584async fn apply_l3_health_action(
5585    spec: &ModuleSpec,
5586    runtime: &SupervisorRuntimeConfig,
5587    registry: &Registry,
5588    process_liveness: &SupervisorProcessLiveness,
5589    snapshot: &SharedSnapshot,
5590    child: &mut Option<SupervisedChild>,
5591    status: SupervisorHealthStatus,
5592    detail: Option<&str>,
5593    action: HealthAction,
5594    now_ms: u64,
5595) {
5596    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5597    match action {
5598        HealthAction::Report => {
5599            info!(
5600                module_id = %spec.module_id,
5601                status = ?status,
5602                detail,
5603                "module reported non-ok health"
5604            );
5605        }
5606        HealthAction::Alert => {
5607            error!(
5608                module_id = %spec.module_id,
5609                status = ?status,
5610                detail,
5611                "module health alert"
5612            );
5613        }
5614        HealthAction::Restart => {
5615            if let Err(err) = health_restart_child(
5616                spec,
5617                runtime,
5618                registry,
5619                process_liveness,
5620                snapshot,
5621                child,
5622                status,
5623                detail,
5624                now_ms,
5625            )
5626            .await
5627            {
5628                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5629            }
5630        }
5631    }
5632}
5633
5634#[allow(clippy::too_many_arguments)]
5635async fn health_restart_child(
5636    spec: &ModuleSpec,
5637    runtime: &SupervisorRuntimeConfig,
5638    registry: &Registry,
5639    process_liveness: &SupervisorProcessLiveness,
5640    snapshot: &SharedSnapshot,
5641    child: &mut Option<SupervisedChild>,
5642    status: SupervisorHealthStatus,
5643    detail: Option<&str>,
5644    now_ms: u64,
5645) -> Result<(), SuperviseError> {
5646    let (enabled, schedule) = {
5647        let mut state = lock_snapshot(snapshot)?;
5648        let enabled = state.enabled;
5649        let schedule = if enabled {
5650            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5651        } else {
5652            None
5653        };
5654        (enabled, schedule)
5655    };
5656
5657    if !enabled {
5658        return Err(SuperviseError::Disabled {
5659            module_id: spec.module_id.clone(),
5660        });
5661    }
5662
5663    if schedule.is_none() {
5664        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5665        error!(
5666            module_id = %spec.module_id,
5667            status = ?status,
5668            detail,
5669            max_restarts = runtime.restart_policy.max_restarts,
5670            window_secs = runtime.restart_policy.window.as_secs(),
5671            reason = %runtime.restart_policy.budget_exhausted_detail(),
5672            "health restart budget exhausted; marking module failed"
5673        );
5674        let stop_notice = begin_forwarding_drain_if_configured(
5675            spec,
5676            runtime,
5677            registry,
5678            snapshot,
5679            Some(true),
5680            RouteCloseReason::Disable,
5681        )
5682        .await?;
5683        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5684            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5685        })?;
5686        let result = drain_optional_child(
5687            &spec.module_id,
5688            spec.protocol,
5689            stop_notice,
5690            registry,
5691            runtime.forwarding.as_deref(),
5692            snapshot,
5693            &runtime.terminal_ring,
5694            &runtime.spawn_events,
5695            child,
5696            runtime.drain_timeout,
5697            ModuleState::Failed,
5698            Some(true),
5699            &runtime.failure_context(true),
5700        )
5701        .await;
5702        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5703            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5704                state.failed_retry.schedule(&spec.module_id);
5705            }
5706        })?;
5707        result?;
5708        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5709        return Ok(());
5710    }
5711
5712    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5713    let mut restart_count = 0;
5714    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5715        restart_count = state.crash_restarts.len();
5716        state.state = ModuleState::Unresponsive;
5717        state.health.status = status;
5718        state.health.last_action = Some(HealthAction::Restart.to_string());
5719        state.health.last_action_ms = Some(now_ms);
5720    })?;
5721    warn!(
5722        module_id = %spec.module_id,
5723        status = ?status,
5724        detail,
5725        restart_count,
5726        restart_in_window = schedule.restart_in_window,
5727        delay_ms = schedule.delay.as_millis() as u64,
5728        "health-triggered module restart"
5729    );
5730
5731    let stop_notice = begin_forwarding_drain_if_configured(
5732        spec,
5733        runtime,
5734        registry,
5735        snapshot,
5736        Some(true),
5737        RouteCloseReason::Restart,
5738    )
5739    .await?;
5740    drain_optional_child(
5741        &spec.module_id,
5742        spec.protocol,
5743        stop_notice,
5744        registry,
5745        runtime.forwarding.as_deref(),
5746        snapshot,
5747        &runtime.terminal_ring,
5748        &runtime.spawn_events,
5749        child,
5750        runtime.drain_timeout,
5751        ModuleState::Restarting,
5752        Some(true),
5753        &runtime.failure_context(true),
5754    )
5755    .await?;
5756    schedule_respawn(
5757        runtime,
5758        snapshot,
5759        &spec.module_id,
5760        schedule.delay,
5761        RespawnKind::Spawn,
5762        true,
5763    )
5764}
5765
5766fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5767    if let Some(reply) = runtime
5768        .deferred_reload_reply
5769        .lock()
5770        .unwrap_or_else(|p| p.into_inner())
5771        .take()
5772    {
5773        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5774            module_id: module_id.to_string(),
5775            reason: reason.to_string(),
5776        }));
5777    }
5778}
5779
5780fn schedule_respawn(
5781    runtime: &SupervisorRuntimeConfig,
5782    snapshot: &SharedSnapshot,
5783    module_id: &str,
5784    delay: Duration,
5785    kind: RespawnKind,
5786    daemon_initiated: bool,
5787) -> Result<(), SuperviseError> {
5788    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5789    update_snapshot(snapshot, Some(module_id), |state| {
5790        state.respawn_pending = true
5791    })?;
5792    *runtime
5793        .scheduled_respawn
5794        .lock()
5795        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5796        deadline: Instant::now() + delay,
5797        kind,
5798        daemon_initiated,
5799    });
5800    Ok(())
5801}
5802
5803fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5804    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5805        state.health.last_action = Some(action);
5806        state.health.last_action_ms = Some(now_ms);
5807    });
5808}
5809
5810fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5811    match status {
5812        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5813        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5814        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5815    }
5816}
5817
5818/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5819/// returned to every `supervisor.list` and `supervisor.health` caller.
5820///
5821/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5822/// path: that request exists to return a module's complete metrics object, and
5823/// `ck health <module-id>` documents it as the way to see what the cached view
5824/// truncates. The asymmetry is the feature.
5825///
5826/// So a new caller must decide which side it is on rather than assume the cap is
5827/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5828/// the truncation that path exists to avoid.
5829fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5830    let metrics = metrics?;
5831    match serde_json::to_vec(&metrics) {
5832        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5833            "truncated": true,
5834            "original_bytes": encoded.len(),
5835        })),
5836        Ok(_) | Err(_) => Some(metrics),
5837    }
5838}
5839
5840/// Spread health probes so a fleet-wide restart does not converge them.
5841///
5842/// The delay is derived from the module id and probe index rather than a random
5843/// source, so it is deterministic per module: a module keeps its own offset
5844/// across daemon restarts instead of re-rolling into a collision.
5845fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5846    if cadence.is_zero() {
5847        return Duration::ZERO;
5848    }
5849    let cadence_ms = cadence.as_millis() as u64;
5850    // This early return is REDUNDANT, deliberately, and a mutation run will show
5851    // it surviving removal. Recording why here so the next person to notice does
5852    // not have to re-derive it:
5853    //
5854    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5855    //   a zero cadence and builds the Duration from whole milliseconds, so a
5856    //   sub-millisecond cadence cannot come from config.
5857    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5858    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5859    //   -- exactly what this returns.
5860    //
5861    // Kept as a guard against a future widening of the config parser (accepting
5862    // microseconds, say), which would make the sub-millisecond case reachable.
5863    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5864    // divides by zero. Remove this and nothing changes.
5865    if cadence_ms == 0 {
5866        return cadence;
5867    }
5868    // Note that this never returns less than one cadence, including for the FIRST
5869    // probe. So a freshly registered module reports health `unknown` for a full
5870    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5871    // ready to answer.
5872    //
5873    // That is a property of the supervisor's schedule, not of any module: an
5874    // operator watching a restart sees `unknown` and cannot tell it from a module
5875    // that is slow to warm. Measured on two unrelated modules, both flipping to
5876    // `ok` between 22s and 32s after restart.
5877    //
5878    // Left as-is because spreading the first probe is what keeps a fleet-wide
5879    // restart from firing fourteen simultaneous probes into a cold machine. The
5880    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5881    // that thundering herd for a faster first reading.
5882    let jitter_span = (cadence_ms / 10).max(1);
5883    let hash = module_id.as_bytes().iter().fold(
5884        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5885        |acc, byte| {
5886            acc.wrapping_mul(1099511628211)
5887                .wrapping_add(u64::from(*byte))
5888        },
5889    );
5890    cadence + Duration::from_millis(hash % jitter_span)
5891}
5892
5893#[cfg(test)]
5894mod tests {
5895    use super::*;
5896
5897    #[test]
5898    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5899        let handle = SupervisorHandle::new();
5900        let module_id = "readded-tombstone";
5901        handle.record_rescan_removal(module_id);
5902        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5903
5904        handle.apply_identity_configuration(&ModuleSpec {
5905            module_id: module_id.to_string(),
5906            program: PathBuf::from("/test/module"),
5907            args: Vec::new(),
5908            env: Vec::new(),
5909            reserved: false,
5910            reserved_prefixes: Vec::new(),
5911            protocol: ModuleProtocol::Subc,
5912            overlap: Default::default(),
5913            carry_containment: false,
5914        });
5915
5916        assert!(
5917            handle.removal_tombstone_age_ms(module_id).is_none(),
5918            "a re-added module must not retain a stale removal tombstone"
5919        );
5920    }
5921
5922    /// What one module's owner looked like from the control plane at the
5923    /// instant after its first process was spawned.
5924    #[derive(Debug, PartialEq, Eq)]
5925    struct OwnerInSpawnWindow {
5926        module_id: String,
5927        configured: bool,
5928        on_roster: bool,
5929        admission_refusal: Option<&'static str>,
5930    }
5931
5932    /// A supervised module's process can connect, register, sync its scopes
5933    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5934    /// the module on the roster until afterward; configured supervision does
5935    /// so before spawning. In both cases the owner must
5936    /// already read as configured, so a scoped `route.open` against it is
5937    /// refused as retryable `scope_not_synced` and not as terminal
5938    /// `scope_not_live` ("will never sync").
5939    ///
5940    /// The hook runs just after each initial spawn attempt, before returning to
5941    /// the caller, so no race with a real child's registration is needed.
5942    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5943    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5944        use crate::scopes::ScopeTable;
5945        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5946
5947        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5948        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5949            module_id: module_id.to_string(),
5950            program,
5951            args: Vec::new(),
5952            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5953                .into_iter()
5954                .map(|key| (key.to_string(), dir.path().display().to_string()))
5955                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5956                .collect(),
5957            reserved: false,
5958            reserved_prefixes: Vec::new(),
5959            protocol: ModuleProtocol::Subc,
5960            overlap: Default::default(),
5961            carry_containment: false,
5962        };
5963        let live = super::terminal_history_tests::fake_aft_stub_path();
5964        let missing = dir.path().join("definitely-missing-module");
5965
5966        let handle = SupervisorHandle::new();
5967        let mut supervisor =
5968            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5969                .with_handle(handle.clone());
5970        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5971        let hook_handle = handle.clone();
5972        let hook_observed = Arc::clone(&observed);
5973        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5974            // Exactly what the control plane computes for a scoped route.open
5975            // naming this module as the owner of a scope it has not synced.
5976            let configured = hook_handle.is_configured(module_id);
5977            let selector = ScopeSelector::new(
5978                Principal::Reserved {
5979                    module_id: module_id.to_string(),
5980                },
5981                "s".to_string(),
5982                Some(1),
5983            );
5984            let carrier = Principal::Reserved {
5985                module_id: "carrier".to_string(),
5986            };
5987            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5988                .admit(&carrier, module_id, &selector, configured)
5989            {
5990                Ok(_) => None,
5991                Err(refusal) => Some(refusal.code),
5992            };
5993            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5994                module_id: module_id.to_string(),
5995                configured,
5996                on_roster: hook_handle.get(module_id).is_some(),
5997                admission_refusal,
5998            });
5999        })));
6000
6001        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
6002        let configured = supervisor
6003            .supervise_configured(stub("configured", live.clone()), true)
6004            .unwrap();
6005        let with_health = supervisor
6006            .supervise_configured_with_health(
6007                stub("with-health", live.clone()),
6008                true,
6009                HealthConfig::default(),
6010                None,
6011                RestartPolicy::default(),
6012            )
6013            .unwrap();
6014        // The failed-spawn path still puts the module on the roster (as
6015        // failed), so it is configured throughout.
6016        let failed = supervisor
6017            .supervise_configured_with_health(
6018                stub("failed-spawn", missing.clone()),
6019                true,
6020                HealthConfig::default(),
6021                None,
6022                RestartPolicy::default(),
6023            )
6024            .unwrap();
6025        // A failed plain `spawn` puts nothing on the roster, so its mark is
6026        // taken back once the spawn has failed.
6027        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6028
6029        let expected = [
6030            "plain",
6031            "configured",
6032            "with-health",
6033            "failed-spawn",
6034            "spawn-error",
6035        ]
6036        .into_iter()
6037        .map(|module_id| OwnerInSpawnWindow {
6038            module_id: module_id.to_string(),
6039            configured: true,
6040            on_roster: !matches!(module_id, "plain" | "spawn-error"),
6041            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6042        })
6043        .collect::<Vec<_>>();
6044        assert_eq!(*observed.lock().unwrap(), expected);
6045
6046        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6047            assert!(
6048                handle.get(module_id).is_some(),
6049                "{module_id} is on the roster"
6050            );
6051            assert!(
6052                handle.is_configured(module_id),
6053                "{module_id} stays configured"
6054            );
6055        }
6056        assert!(handle.get("spawn-error").is_none());
6057        assert!(
6058            !handle.is_configured("spawn-error"),
6059            "a plain spawn that failed must not leave its module marked configured"
6060        );
6061
6062        // Leaving the roster clears the mark with it.
6063        handle.retire("failed-spawn");
6064        assert!(!handle.is_configured("failed-spawn"));
6065
6066        for module in [plain, configured, with_health] {
6067            module.stop().await.unwrap();
6068        }
6069        drop(failed);
6070    }
6071
6072    /// A configured module reads as starting until the supervisor has admitted
6073    /// its first process (exec confirmed, identity durably recorded), even
6074    /// though that process is already running by then and can register over
6075    /// HELLO, which is what lists it in catalog.list. So a client that needs
6076    /// the supervisor state must wait for that state; seeing the module in
6077    /// the catalog does not mean supervisor.list already says running.
6078    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6079    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6080        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6081        let spec = ModuleSpec {
6082            module_id: "spawned-not-admitted".to_string(),
6083            program: super::terminal_history_tests::fake_aft_stub_path(),
6084            args: Vec::new(),
6085            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6086                .into_iter()
6087                .map(|key| (key.to_string(), dir.path().display().to_string()))
6088                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6089                .collect(),
6090            reserved: false,
6091            reserved_prefixes: Vec::new(),
6092            protocol: ModuleProtocol::Subc,
6093            overlap: Default::default(),
6094            carry_containment: false,
6095        };
6096        let handle = SupervisorHandle::new();
6097        let mut supervisor =
6098            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6099                .with_handle(handle.clone());
6100        let observed = Arc::new(Mutex::new(None));
6101        let hook_handle = handle.clone();
6102        let hook_observed = Arc::clone(&observed);
6103        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6104            let state = hook_handle
6105                .get(module_id)
6106                .expect("a configured module is on the roster before it is spawned")
6107                .state()
6108                .unwrap();
6109            *hook_observed.lock().unwrap() = Some(state);
6110        })));
6111
6112        let module = supervisor.supervise_configured(spec, true).unwrap();
6113        assert_eq!(
6114            *observed.lock().unwrap(),
6115            Some(ModuleState::Starting),
6116            "between the spawn and its admission the supervisor still reports starting"
6117        );
6118        assert_eq!(module.state().unwrap(), ModuleState::Running);
6119        module.stop().await.unwrap();
6120    }
6121
6122    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6123        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6124        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6125            snapshot.process_alive = true;
6126            snapshot.pid = Some(41);
6127            snapshot.spawned_at_ms = Some(42);
6128            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6129            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6130            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6131                device: 43,
6132                inode: 44,
6133                #[cfg(windows)]
6134                windows_image: None,
6135            });
6136        })
6137        .unwrap();
6138        snapshot
6139    }
6140
6141    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6142        let snapshot = lock_snapshot(snapshot).unwrap();
6143        assert!(!snapshot.process_alive);
6144        assert_eq!(snapshot.pid, None);
6145        assert_eq!(snapshot.spawned_at_ms, None);
6146        assert_eq!(snapshot.spawned_from, None);
6147        assert_eq!(snapshot.launch_spec_digest, None);
6148        assert_eq!(snapshot.spawned_file_identity, None);
6149    }
6150
6151    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6152    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6153        let supervisor =
6154            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6155        let mut runtime = supervisor.runtime_config();
6156        runtime.test_seed_stale_facts_before_enable_spawn = true;
6157        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6158        let mut child = None;
6159        let spec = ModuleSpec {
6160            module_id: "failed-enable-clears-facts".to_string(),
6161            program: PathBuf::from("/definitely/missing/failed-enable-module"),
6162            args: Vec::new(),
6163            env: Vec::new(),
6164            reserved: false,
6165            reserved_prefixes: Vec::new(),
6166            protocol: ModuleProtocol::Subc,
6167            overlap: Default::default(),
6168            carry_containment: false,
6169        };
6170
6171        let result = set_child_enabled(
6172            &spec,
6173            &runtime,
6174            &supervisor.registry,
6175            &supervisor.process_liveness,
6176            &snapshot,
6177            &mut child,
6178            true,
6179            &runtime.failure_context(false),
6180        )
6181        .await;
6182
6183        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6184        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6185        assert_snapshot_process_facts_cleared(&snapshot);
6186    }
6187
6188    #[tokio::test]
6189    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6190        let supervisor =
6191            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6192        let runtime = supervisor.runtime_config();
6193        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6194            ModuleState::Restarting,
6195            true,
6196        )));
6197        let spec = ModuleSpec {
6198            module_id: "start-stranded-restarting".to_string(),
6199            program: super::terminal_history_tests::fake_aft_stub_path(),
6200            args: Vec::new(),
6201            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6202            reserved: false,
6203            reserved_prefixes: Vec::new(),
6204            protocol: ModuleProtocol::None,
6205            overlap: Default::default(),
6206            carry_containment: false,
6207        };
6208        let mut child = None;
6209        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6210        assert!(!super::set_child_enabled(
6211            &spec,
6212            &runtime,
6213            &Registry::default(),
6214            &supervisor.process_liveness,
6215            &snapshot,
6216            &mut child,
6217            true,
6218            &runtime.failure_context(false),
6219        )
6220        .await
6221        .unwrap());
6222        assert!(child.is_none());
6223        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6224        assert!(super::set_child_enabled(
6225            &spec,
6226            &runtime,
6227            &Registry::default(),
6228            &supervisor.process_liveness,
6229            &snapshot,
6230            &mut child,
6231            true,
6232            &runtime.failure_context(false),
6233        )
6234        .await
6235        .unwrap());
6236        assert_eq!(
6237            lock_snapshot(&snapshot).unwrap().state,
6238            ModuleState::Running
6239        );
6240        let mut child = child.unwrap();
6241        child.start_kill().unwrap();
6242        child.wait().await.unwrap();
6243    }
6244
6245    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6246    async fn failed_reload_spawn_clears_current_process_facts() {
6247        let supervisor =
6248            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6249        let mut runtime = supervisor.runtime_config();
6250        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6251        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6252        let mut child = None;
6253        let spec = ModuleSpec {
6254            module_id: "failed-reload-clears-facts".to_string(),
6255            program: PathBuf::from("/unused/failed-reload-module"),
6256            args: Vec::new(),
6257            env: Vec::new(),
6258            reserved: false,
6259            reserved_prefixes: Vec::new(),
6260            protocol: ModuleProtocol::Subc,
6261            overlap: Default::default(),
6262            carry_containment: false,
6263        };
6264
6265        let result = handle_reload_spawn_failure(
6266            &spec,
6267            &runtime,
6268            &supervisor.process_liveness,
6269            &snapshot,
6270            &mut child,
6271            "forced reload spawn failure".to_string(),
6272        )
6273        .await;
6274
6275        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6276        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6277        assert_snapshot_process_facts_cleared(&snapshot);
6278    }
6279
6280    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6281    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6282        let supervisor =
6283            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6284        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6285        let module = supervisor.supervised_module(
6286            ModuleSpec {
6287                module_id: "drop-clears-facts".to_string(),
6288                program: PathBuf::from("/unused/drop-module"),
6289                args: Vec::new(),
6290                env: Vec::new(),
6291                reserved: false,
6292                reserved_prefixes: Vec::new(),
6293                protocol: ModuleProtocol::Subc,
6294                overlap: Default::default(),
6295                carry_containment: false,
6296            },
6297            supervisor.runtime_config(),
6298            Arc::clone(&snapshot),
6299            None,
6300        );
6301        assert!(!module
6302            .inner
6303            .monitor
6304            .lock()
6305            .unwrap()
6306            .as_ref()
6307            .unwrap()
6308            .is_finished());
6309
6310        drop(module);
6311
6312        assert_eq!(
6313            lock_snapshot(&snapshot).unwrap().state,
6314            ModuleState::Stopped
6315        );
6316        assert_snapshot_process_facts_cleared(&snapshot);
6317    }
6318
6319    #[cfg(unix)]
6320    #[tokio::test]
6321    async fn rescan_preserves_running_protocol_until_respawn() {
6322        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6323        let initial = ModuleSpec {
6324            module_id: "rescan-protocol".into(),
6325            program: PathBuf::from("/bin/sleep"),
6326            args: vec!["60".into()],
6327            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6328                .into_iter()
6329                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6330                .collect(),
6331            reserved: false,
6332            reserved_prefixes: vec![],
6333            protocol: ModuleProtocol::None,
6334            overlap: Default::default(),
6335            carry_containment: false,
6336        };
6337        let supervisor =
6338            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6339        let module = supervisor.spawn(initial.clone()).unwrap();
6340        assert!(module.status().unwrap().live);
6341        let mut next = initial;
6342        next.protocol = ModuleProtocol::Subc;
6343        module
6344            .update_configuration(next.clone(), HealthConfig::default(), None)
6345            .await
6346            .unwrap();
6347        assert!(
6348            module.status().unwrap().live,
6349            "rescan must not require HELLO from the old non-wire process"
6350        );
6351        let runtime = supervisor.runtime_config();
6352        let action = on_child_exit(
6353            &next,
6354            RestartPolicy::default(),
6355            &supervisor.registry,
6356            &module.inner.snapshot,
6357            &runtime.terminal_ring,
6358            &runtime.spawn_events,
6359            &runtime.child_roster,
6360            ExitReport {
6361                kind: ExitKind::Clean,
6362                code: Some(0),
6363                signal: None,
6364                at_ms: unix_ms_now(),
6365            },
6366            true,
6367        )
6368        .await;
6369        assert!(
6370            matches!(action, NextAction::Restart { .. }),
6371            "the old non-wire process's clean exit must restart"
6372        );
6373        module.drain().await.unwrap();
6374    }
6375
6376    #[cfg(unix)]
6377    #[tokio::test]
6378    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6379        use std::os::unix::fs::PermissionsExt;
6380        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6381        let script = dir.join("module.sh");
6382        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6383        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6384        let record_path = dir.join("live-children.json");
6385        let supervisor =
6386            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6387                .with_live_children_record(&record_path);
6388        for (program, args) in [
6389            (PathBuf::from("sleep"), vec!["60".into()]),
6390            (script, vec![]),
6391        ] {
6392            let spec = ModuleSpec {
6393                module_id: "image-identity".into(),
6394                program: program.clone(),
6395                args,
6396                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6397                    .into_iter()
6398                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6399                    .collect(),
6400                reserved: false,
6401                reserved_prefixes: vec![],
6402                protocol: ModuleProtocol::None,
6403                overlap: Default::default(),
6404                carry_containment: false,
6405            };
6406            let module = supervisor.spawn(spec).unwrap();
6407            #[cfg(target_os = "macos")]
6408            {
6409                // SETEXEC confirmation is asynchronous; the orphan record must
6410                // identify the final image, never the intermediate trampoline.
6411                let deadline = Instant::now() + Duration::from_secs(5);
6412                while crate::live_children::read_record(&record_path)
6413                    .unwrap()
6414                    .iter()
6415                    .all(|entry| entry.executable.is_none())
6416                {
6417                    assert!(Instant::now() < deadline, "module image was not confirmed");
6418                    tokio::time::sleep(Duration::from_millis(5)).await;
6419                }
6420            }
6421            let entry = crate::live_children::read_record(&record_path)
6422                .unwrap()
6423                .pop()
6424                .unwrap();
6425            let observed = subc_os::Process::open(entry.pid)
6426                .unwrap()
6427                .unwrap()
6428                .observe()
6429                .unwrap();
6430            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6431            module.drain().await.unwrap();
6432            assert_eq!(
6433                verdict,
6434                crate::live_children::IdentityVerdict::Matches,
6435                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6436            );
6437        }
6438    }
6439
6440    #[cfg(unix)]
6441    fn http_fixture(
6442        dir: &std::path::Path,
6443        url: &str,
6444        threshold: u32,
6445    ) -> crate::daemon_config::ConfiguredModule {
6446        let path = dir.join("subc.jsonc");
6447        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6448            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6449            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6450            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6451            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6452        }}}).to_string()).unwrap();
6453        crate::daemon_config::load(&path)
6454            .unwrap()
6455            .unwrap()
6456            .modules
6457            .pop()
6458            .unwrap()
6459    }
6460
6461    #[cfg(unix)]
6462    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6463        timeout(Duration::from_secs(5), async {
6464            loop {
6465                if module.status().unwrap().health.status == status {
6466                    break;
6467                }
6468                sleep(Duration::from_millis(5)).await;
6469            }
6470        })
6471        .await
6472        .unwrap_or_else(|_| {
6473            panic!(
6474                "expected {status:?}, got {:?}",
6475                module.status().unwrap().health
6476            )
6477        });
6478    }
6479
6480    #[cfg(unix)]
6481    #[tokio::test]
6482    async fn http_health_status_flips_ok_failing_ok() {
6483        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6484        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6485        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6486        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6487        let serving_status = status.clone();
6488        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6489        let server = tokio::spawn(async move {
6490            loop {
6491                let (mut stream, _) = listener.accept().await.unwrap();
6492                let mut request = [0u8; 2048];
6493                let count = stream.read(&mut request).await.unwrap();
6494                assert!(count > 0, "a probe must send an HTTP request");
6495                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6496                let body = if code == 200 {
6497                    "ready"
6498                } else {
6499                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6500                };
6501                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6502                let _ = stream.write_all(response.as_bytes()).await;
6503            }
6504        });
6505        let configured = http_fixture(&dir, &url, 1000);
6506        let module =
6507            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6508                .supervise_configured_with_health(
6509                    configured.module_spec(),
6510                    true,
6511                    configured.health,
6512                    configured.drain_timeout_ms,
6513                    configured.restart,
6514                )
6515                .unwrap();
6516        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6517        status.store(503, std::sync::atomic::Ordering::SeqCst);
6518        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6519        assert!(module
6520            .status()
6521            .unwrap()
6522            .health
6523            .detail
6524            .unwrap()
6525            .contains("scratch failure"));
6526        status.store(200, std::sync::atomic::Ordering::SeqCst);
6527        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6528        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6529        let before = module.status().unwrap();
6530        let (spec, mut health) = module.configuration().unwrap();
6531        health.http = None;
6532        module
6533            .update_configuration(spec.clone(), health.clone(), Some(10))
6534            .await
6535            .unwrap();
6536        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6537        health.http = Some(url);
6538        module
6539            .update_configuration(spec, health, Some(10))
6540            .await
6541            .unwrap();
6542        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6543        assert_eq!(
6544            module.status().unwrap().pid,
6545            before.pid,
6546            "changing a probe must apply live, not restart its process"
6547        );
6548        let (spec, mut health) = module.configuration().unwrap();
6549        health.failure_threshold = 2;
6550        module
6551            .update_configuration(spec, health, Some(10))
6552            .await
6553            .unwrap();
6554        status.store(503, std::sync::atomic::Ordering::SeqCst);
6555        timeout(Duration::from_secs(5), async {
6556            while module.status().unwrap().spawn_generation == before.spawn_generation {
6557                sleep(Duration::from_millis(5)).await;
6558            }
6559        })
6560        .await
6561        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6562        module.drain().await.unwrap();
6563        server.abort();
6564    }
6565
6566    #[cfg(unix)]
6567    #[tokio::test]
6568    async fn http_health_replacement_recovers_after_one_restart() {
6569        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6570        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6571        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6572        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6573        let mut configured = http_fixture(&dir, &url, 2);
6574        configured.restart.max_restarts = 2;
6575        let module =
6576            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6577                .supervise_configured_with_health(
6578                    configured.module_spec(),
6579                    true,
6580                    configured.health,
6581                    configured.drain_timeout_ms,
6582                    configured.restart,
6583                )
6584                .unwrap();
6585        let first_generation = module.status().unwrap().spawn_generation;
6586        let observed = module.clone();
6587        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6588        let server = tokio::spawn(async move {
6589            let mut generation = first_generation;
6590            let mut probes = 0;
6591            loop {
6592                let (mut stream, _) = listener.accept().await.unwrap();
6593                let mut request = [0; 2048];
6594                assert!(stream.read(&mut request).await.unwrap() > 0);
6595                let current = observed.status().unwrap().spawn_generation;
6596                if current != generation {
6597                    generation = current;
6598                    probes = 0;
6599                }
6600                probes += 1;
6601                // The original process never recovers. Each replacement has one
6602                // transient miss, then answers successfully for its remaining life.
6603                let healthy = generation != first_generation && probes > 1;
6604                let code = if healthy { 200 } else { 503 };
6605                let response = format!(
6606                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6607                );
6608                let _ = stream.write_all(response.as_bytes()).await;
6609                if healthy {
6610                    healthy_tx.send(()).await.unwrap();
6611                }
6612            }
6613        });
6614        timeout(Duration::from_secs(5), async {
6615            let mut healthy_probes = 0;
6616            loop {
6617                if module.state().unwrap() == ModuleState::Failed {
6618                    break;
6619                }
6620                while healthy_rx.try_recv().is_ok() {
6621                    healthy_probes += 1;
6622                }
6623                if healthy_probes >= 3
6624                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6625                {
6626                    break;
6627                }
6628                sleep(Duration::from_millis(5)).await;
6629            }
6630        })
6631        .await
6632        .expect("the replacement must settle healthy without exhausting its restart budget");
6633        let status = module.status().unwrap();
6634        module.drain().await.unwrap();
6635        server.abort();
6636        assert_eq!(status.state, ModuleState::Running);
6637        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6638        assert_eq!(status.health.consecutive_failures, 0);
6639        assert_eq!(status.lifetime_restarts, 1);
6640        assert_eq!(status.restart_count, 1);
6641    }
6642
6643    #[cfg(unix)]
6644    #[tokio::test]
6645    async fn http_health_sustained_failure_exhausts_restart_budget() {
6646        use std::collections::BTreeMap;
6647        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6648        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6649        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6650        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6651        let mut configured = http_fixture(&dir, &url, 2);
6652        configured.restart.max_restarts = 2;
6653        let module =
6654            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6655                .supervise_configured_with_health(
6656                    configured.module_spec(),
6657                    true,
6658                    configured.health,
6659                    configured.drain_timeout_ms,
6660                    configured.restart,
6661                )
6662                .unwrap();
6663        let observed = module.clone();
6664        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6665        let served = probes.clone();
6666        let server = tokio::spawn(async move {
6667            loop {
6668                let (mut stream, _) = listener.accept().await.unwrap();
6669                let mut request = [0; 2048];
6670                assert!(stream.read(&mut request).await.unwrap() > 0);
6671                *served
6672                    .lock()
6673                    .unwrap()
6674                    .entry(observed.status().unwrap().spawn_generation)
6675                    .or_default() += 1;
6676                let _ = stream
6677                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6678                    .await;
6679            }
6680        });
6681        timeout(Duration::from_secs(5), async {
6682            while module.state().unwrap() != ModuleState::Failed {
6683                sleep(Duration::from_millis(5)).await;
6684            }
6685        })
6686        .await
6687        .expect("sustained failures must still exhaust the health restart budget");
6688        let status = module.status().unwrap();
6689        server.abort();
6690        assert_eq!(status.restart_count, 2);
6691        assert_eq!(status.lifetime_restarts, 2);
6692        assert!(!status.process_alive);
6693        assert_eq!(
6694            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6695            vec![2, 2, 2]
6696        );
6697    }
6698
6699    #[cfg(unix)]
6700    #[tokio::test]
6701    async fn spawn_clears_the_previous_process_health() {
6702        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6703        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6704        configured.health.cadence = Duration::from_secs(3600);
6705        let module =
6706            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6707                .supervise_configured_with_health(
6708                    configured.module_spec(),
6709                    false,
6710                    configured.health,
6711                    configured.drain_timeout_ms,
6712                    configured.restart,
6713                )
6714                .unwrap();
6715        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6716            state.health.status = SupervisorHealthStatus::Unresponsive;
6717            state.health.consecutive_failures = 3;
6718            state.health.detail = Some("old process refused connections".into());
6719            state.health.metrics = Some(serde_json::json!({"old": true}));
6720            state.health.last_probe_ms = Some(123);
6721            state.health.last_action = Some("restart".into());
6722            state.health.last_action_ms = Some(124);
6723        })
6724        .unwrap();
6725        module.set_enabled(true).await.unwrap();
6726        let health = module.status().unwrap().health;
6727        module.drain().await.unwrap();
6728        assert_eq!(health.consecutive_failures, 0);
6729        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6730        assert_eq!(health.detail, None);
6731        assert_eq!(health.metrics, None);
6732        assert_eq!(health.last_probe_ms, None);
6733        assert_eq!(health.last_action.as_deref(), Some("restart"));
6734        assert_eq!(health.last_action_ms, Some(124));
6735    }
6736
6737    #[cfg(unix)]
6738    #[tokio::test]
6739    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6740        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6741        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6742        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6743        drop(unused);
6744        let configured = http_fixture(&dir, &url, 2);
6745        let module =
6746            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6747                .supervise_configured_with_health(
6748                    configured.module_spec(),
6749                    true,
6750                    configured.health,
6751                    configured.drain_timeout_ms,
6752                    configured.restart,
6753                )
6754                .unwrap();
6755        let before = module.status().unwrap().spawn_generation;
6756        timeout(Duration::from_secs(5), async {
6757            loop {
6758                let status = module.status().unwrap();
6759                if status.spawn_generation > before {
6760                    assert!(status.lifetime_restarts > 0);
6761                    break;
6762                }
6763                sleep(Duration::from_millis(5)).await;
6764            }
6765        })
6766        .await
6767        .expect("sustained HTTP refusal must trigger the health restart policy");
6768        module.drain().await.unwrap();
6769    }
6770
6771    #[cfg(unix)]
6772    #[tokio::test]
6773    async fn http_health_timeout_honours_deadline() {
6774        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6775        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6776        let server = tokio::spawn(async move {
6777            let _held = listener.accept().await.unwrap();
6778            std::future::pending::<()>().await;
6779        });
6780        let error = timeout(
6781            Duration::from_secs(1),
6782            probe_http_health(&url, Duration::from_millis(10)),
6783        )
6784        .await
6785        .expect("the probe must enforce its own deadline")
6786        .unwrap_err();
6787        server.abort();
6788        assert!(error.to_string().contains("timed out"));
6789    }
6790
6791    #[cfg(unix)]
6792    #[tokio::test]
6793    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6794        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6795        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6796        let url = format!(
6797            "http://localhost:{}/healthz",
6798            listener.local_addr().unwrap().port()
6799        );
6800        let server = tokio::spawn(async move {
6801            let (mut stream, _) = listener.accept().await.unwrap();
6802            let mut request = [0u8; 2048];
6803            assert!(stream.read(&mut request).await.unwrap() > 0);
6804            stream
6805                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6806                .await
6807                .unwrap();
6808        });
6809        // The deadline only bounds a hang. A probe that never tried the IPv6
6810        // address would be refused on 127.0.0.1 and fail at once, so a longer
6811        // deadline does not weaken the assertion; one second timed out under a
6812        // loaded parallel test run.
6813        assert_eq!(
6814            probe_http_health(&url, Duration::from_secs(10))
6815                .await
6816                .unwrap()
6817                .status,
6818            HealthStatus::Ok
6819        );
6820        server.await.unwrap();
6821    }
6822
6823    #[cfg(unix)]
6824    #[tokio::test]
6825    async fn http_health_timeout_keeps_partial_status_and_body() {
6826        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6827        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6828        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6829        let server = tokio::spawn(async move {
6830            let (mut stream, _) = listener.accept().await.unwrap();
6831            let mut request = [0u8; 2048];
6832            assert!(stream.read(&mut request).await.unwrap() > 0);
6833            stream
6834                .write_all(
6835                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6836                )
6837                .await
6838                .unwrap();
6839            std::future::pending::<()>().await;
6840        });
6841        let error = probe_http_health(&url, Duration::from_secs(1))
6842            .await
6843            .unwrap_err()
6844            .to_string();
6845        server.abort();
6846        assert!(
6847            error.contains("timed out")
6848                && error.contains("503 Unavailable")
6849                && error.contains("partial diagnostic"),
6850            "{error}"
6851        );
6852    }
6853
6854    #[cfg(unix)]
6855    #[tokio::test]
6856    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6857        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6858        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6859        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6860        let server = tokio::spawn(async move {
6861            let (mut stream, _) = listener.accept().await.unwrap();
6862            let mut request = [0u8; 2048];
6863            assert!(stream.read(&mut request).await.unwrap() > 0);
6864            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6865            let response = format!(
6866                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6867                body.len()
6868            );
6869            stream.write_all(response.as_bytes()).await.unwrap();
6870        });
6871        let error = probe_http_health(&url, Duration::from_secs(1))
6872            .await
6873            .unwrap_err()
6874            .to_string();
6875        server.await.unwrap();
6876        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6877        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6878        assert!(!error.contains("not-in-diagnostic"));
6879    }
6880
6881    #[cfg(unix)]
6882    #[tokio::test]
6883    async fn http_health_real_nats_server_monitoring() {
6884        if std::process::Command::new("nats-server")
6885            .arg("--version")
6886            .env("XDG_DATA_HOME", std::env::temp_dir())
6887            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6888            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6889            .output()
6890            .is_err()
6891        {
6892            eprintln!(
6893                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6894            );
6895            return;
6896        }
6897        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6898        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6899        let port = monitor.local_addr().unwrap().port();
6900        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6901        let client_port = client.local_addr().unwrap().port();
6902        let config = dir.join("server.conf");
6903        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6904        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6905        configured.program = PathBuf::from("nats-server");
6906        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6907        drop(monitor);
6908        drop(client);
6909        let module =
6910            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6911                .supervise_configured_with_health(
6912                    configured.module_spec(),
6913                    true,
6914                    configured.health,
6915                    configured.drain_timeout_ms,
6916                    configured.restart,
6917                )
6918                .unwrap();
6919        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6920        module.drain().await.unwrap();
6921    }
6922
6923    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6924    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6925        let supervisor =
6926            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6927        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6928        let initial = ModuleSpec {
6929            module_id: "rescan-preserves-spawn-facts".to_string(),
6930            program: PathBuf::from("/spawned/module"),
6931            args: Vec::new(),
6932            env: Vec::new(),
6933            reserved: false,
6934            reserved_prefixes: Vec::new(),
6935            protocol: ModuleProtocol::Subc,
6936            overlap: Default::default(),
6937            carry_containment: false,
6938        };
6939        let module = supervisor.supervised_module(
6940            initial.clone(),
6941            supervisor.runtime_config(),
6942            snapshot,
6943            None,
6944        );
6945        let before = module.status().unwrap();
6946        let mut replacement = initial;
6947        replacement.program = PathBuf::from("/rescanned/replacement-module");
6948
6949        module
6950            .update_configuration(replacement, HealthConfig::default(), None)
6951            .await
6952            .unwrap();
6953
6954        let after = module.status().unwrap();
6955        assert_eq!(after.pid, before.pid);
6956        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6957        assert_eq!(after.spawned_from, before.spawned_from);
6958        drop(module);
6959    }
6960}
6961
6962fn unix_ms_now() -> u64 {
6963    SystemTime::now()
6964        .duration_since(UNIX_EPOCH)
6965        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6966        .unwrap_or(0)
6967}
6968
6969async fn supervise_loop(
6970    mut spec: ModuleSpec,
6971    mut runtime: SupervisorRuntimeConfig,
6972    registry: Arc<Registry>,
6973    process_liveness: Arc<SupervisorProcessLiveness>,
6974    snapshot: SharedSnapshot,
6975    mut child: Option<SupervisedChild>,
6976    mut commands: mpsc::Receiver<SupervisorCommand>,
6977) {
6978    let mut health_probe = HealthProbeRuntime::default();
6979    // Registry writes (including embedded callers) notify this module only.
6980    // Subscribe before the first refresh; watch retains changes that arrive
6981    // while commands or probes are running. No polling fallback is needed.
6982    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6983        Ok(changes) => Some(changes),
6984        Err(err) => {
6985            // A poisoned registry cannot accept further writes, so it cannot
6986            // recover via a timer. Keep exit and command handling alive.
6987            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6988            None
6989        }
6990    };
6991    // All restart backoffs run here, including health and operator requests.
6992    // While one is pending the loop serves commands, so disable or drain can
6993    // cancel the replacement without spawning a process just to stop it.
6994    let mut pending_respawn: Option<PendingRespawn> = None;
6995    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6996    // before anything else so a stop that interrupted a swap runs at once.
6997    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6998    let mut previous_instance_recheck = tokio::time::interval_at(
6999        Instant::now() + Duration::from_secs(5),
7000        Duration::from_secs(5),
7001    );
7002    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
7003    loop {
7004        #[cfg(test)]
7005        {
7006            lock_snapshot(&snapshot).unwrap().actor_select = None;
7007        }
7008        #[cfg(target_os = "macos")]
7009        if let Some(active) = child.as_mut() {
7010            active.confirm_privacy_exec().await;
7011        }
7012        if let Some(scheduled) = runtime
7013            .scheduled_respawn
7014            .lock()
7015            .unwrap_or_else(|p| p.into_inner())
7016            .take()
7017        {
7018            runtime.recovery_is_daemon = scheduled.daemon_initiated;
7019            pending_respawn = Some(scheduled);
7020        }
7021        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7022            pending_respawn = None;
7023            cancel_deferred_reload(
7024                &runtime,
7025                &spec.module_id,
7026                "respawn cancelled by a supervisor command",
7027            );
7028        }
7029        if child.is_none()
7030            && pending_respawn.is_none()
7031            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7032        {
7033            cancel_deferred_reload(
7034                &runtime,
7035                &spec.module_id,
7036                "respawn cancelled before a replacement was spawned",
7037            );
7038            let mut stranded = false;
7039            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7040                state.respawn_pending = false;
7041                state.coalesced_restart_pending = false;
7042                if matches!(
7043                    state.state,
7044                    ModuleState::Restarting
7045                        | ModuleState::Starting
7046                        | ModuleState::Draining
7047                        | ModuleState::Unresponsive
7048                ) {
7049                    error!(module_id = %spec.module_id, state = ?state.state,
7050                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7051                    stranded = true;
7052                }
7053            });
7054            if stranded {
7055                fail_snapshot(
7056                    &snapshot,
7057                    Some(&spec.module_id),
7058                    None,
7059                    &runtime.failure_context(runtime.recovery_is_daemon),
7060                );
7061            }
7062        }
7063        if child.is_none()
7064            && (runtime.child_roster.is_closed()
7065                || pending_respawn.is_none()
7066                || !lock_snapshot(&snapshot)
7067                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7068        {
7069            containment::terminate(&runtime.child_roster.containment).await;
7070        }
7071        if let Some(command) = requeued.pop_front() {
7072            if !handle_supervisor_command(
7073                command,
7074                &mut spec,
7075                &mut runtime,
7076                &registry,
7077                &process_liveness,
7078                &snapshot,
7079                &mut child,
7080                &mut commands,
7081                &mut requeued,
7082            )
7083            .await
7084            {
7085                containment::terminate(&runtime.child_roster.containment).await;
7086                return;
7087            }
7088            if child.is_some() || !respawn_still_pending(&snapshot) {
7089                pending_respawn = None;
7090                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7091                    state.respawn_pending = false
7092                });
7093            }
7094            continue;
7095        }
7096        if child.is_some() {
7097            runtime.recovery_is_daemon = true;
7098            #[cfg(test)]
7099            {
7100                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7101            }
7102            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
7103            let active = child.as_mut().expect("child checked above");
7104            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7105                active.registration_deadline = None;
7106            }
7107            let registration_deadline = active.registration_deadline;
7108            let no_checks = runtime.health.http.is_none()
7109                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7110                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7111            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7112                state
7113                    .failed_retry
7114                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
7115            });
7116            let wake_at = match (health_probe.wake_at(), uptime_reset) {
7117                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7118                (probe, None) => probe,
7119            };
7120            let wake_at = match (wake_at, registration_deadline) {
7121                (Some(wake), Some(register)) => Some(wake.min(register)),
7122                (wake, register) => wake.or(register),
7123            };
7124            #[cfg(test)]
7125            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7126            let probe_sleep = supervisor_sleep(wake_at);
7127            tokio::pin!(probe_sleep);
7128            // A `protocol: "none"` child (a plain process that never registers
7129            // over the subc wire, such as nats-server) with no HTTP health
7130            // check has nothing to probe, so it waits only for its exit or a
7131            // supervisor command. A rescan that adds an HTTP check is a command.
7132            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7133            #[cfg(test)]
7134            {
7135                let registration_pending = wire_child
7136                    && registration_changes
7137                        .as_ref()
7138                        .is_some_and(|changes| changes.has_changed().unwrap());
7139                let mut state = lock_snapshot(&snapshot).unwrap();
7140                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7141                    generation: state.spawn_generation,
7142                    turn: state.actor_turns,
7143                    registered_connection: health_probe.registered_connection,
7144                    next_probe_at: health_probe.next_probe_at,
7145                    wake_after,
7146                });
7147            }
7148            let active_child = child.as_mut().expect("child checked above");
7149            tokio::select! {
7150                wait_result = active_child.wait() => {
7151                    if let Ok(mut state) = lock_snapshot(&snapshot) {
7152                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7153                    }
7154                    #[cfg(test)]
7155                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7156                    // Every arm below that gives up on the CHILD must keep the
7157                    // supervision task itself alive (child = None, loop
7158                    // continues into command-serving mode). Returning here
7159                    // closes the command channel, which makes the module
7160                    // permanently unrestartable in-band: a clean child exit
7161                    // of an enabled module once wedged the fleet this way
7162                    // ('supervisor command channel is closed') and required a
7163                    // full daemon restart to recover.
7164                    let exit_report = match wait_result {
7165                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7166                        Err(err) => {
7167                            active_child.drain_stderr(&spec.module_id).await;
7168                            fail_after_wait_error(
7169                                &spec.module_id,
7170                                &snapshot,
7171                                &runtime.terminal_ring,
7172                                &runtime.spawn_events,
7173                                format!("failed to wait for supervised module: {err}"),
7174                            &runtime.failure_context(true),
7175                            );
7176                            untrack_if_registration_released(
7177                                &process_liveness,
7178                                &registry,
7179                                &spec.module_id,
7180                                &snapshot,
7181                            );
7182                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7183                            child = None;
7184                            continue;
7185                        }
7186                    };
7187                    active_child.drain_stderr(&spec.module_id).await;
7188
7189                    let next = on_child_exit(
7190                        &spec,
7191                        runtime.restart_policy,
7192                        &registry,
7193                        &snapshot,
7194                        &runtime.terminal_ring,
7195                        &runtime.spawn_events,
7196                        &runtime.child_roster,
7197                        exit_report,
7198                        true,
7199                    ).await;
7200                    runtime.start_requester = None;
7201                    runtime.pending_activity_restart = false;
7202                    // The exit is recorded, so a daemon shutdown may stop
7203                    // waiting for this child (see `SupervisedChild::wait`).
7204                    let carry = matches!(next, NextAction::Restart { .. })
7205                        && active_child.spawn_failure.is_none()
7206                        && !runtime.child_roster.is_closed();
7207                    if let Err(error) = active_child.finish_exit(carry).await {
7208                        containment::terminate(&runtime.child_roster.containment).await;
7209                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7210                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7211                            &runtime.failure_context(true));
7212                        child = None;
7213                        continue;
7214                    }
7215                    match next {
7216                        NextAction::Stop { registration_released } => {
7217                            if registration_released {
7218                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7219                            }
7220                            child = None;
7221                        }
7222                        NextAction::Restart { schedule } => {
7223                            let delay = schedule.map_or(
7224                                runtime.restart_policy.delay_for_restart(0),
7225                                |schedule| schedule.delay,
7226                            );
7227                            if let Some(schedule) = schedule {
7228                                log_crash_respawn(&spec.module_id, schedule);
7229                            }
7230                            // The exited child is fully recorded at this point,
7231                            // so release it and count the backoff down in the
7232                            // command-serving branch below rather than sleeping
7233                            // here: commands cannot be received from inside this
7234                            // select arm, and an operator disable or drain that
7235                            // arrives during the backoff must cancel the pending
7236                            // respawn instead of waiting for it to spawn first.
7237                            child = None;
7238                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7239                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7240                        }
7241                    }
7242                }
7243                command = commands.recv() => {
7244                    #[cfg(test)]
7245                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7246                    let Some(command) = command else {
7247                        containment::terminate(&runtime.child_roster.containment).await;
7248                        return;
7249                    };
7250                    if !handle_supervisor_command(
7251                        command,
7252                        &mut spec,
7253                        &mut runtime,
7254                        &registry,
7255                        &process_liveness,
7256                        &snapshot,
7257                        &mut child,
7258                        &mut commands,
7259                        &mut requeued,
7260                    ).await {
7261                        containment::terminate(&runtime.child_roster.containment).await;
7262                        return;
7263                    }
7264                }
7265                _ = async {
7266                    match registration_changes.as_mut() {
7267                        Some(changes) => { let _ = changes.changed().await; }
7268                        None => std::future::pending().await,
7269                    }
7270                }, if wire_child => {
7271                    #[cfg(test)]
7272                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7273                }
7274                _ = &mut probe_sleep => {
7275                    #[cfg(test)]
7276                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7277                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7278                        // Recheck after the wake: HELLO, exit or shutdown may
7279                        // have become ready in the same executor tick.
7280                        active_child.registration_deadline = None;
7281                        if !runtime.child_roster.is_closed()
7282                            && !active_child.registered_since_spawn()
7283                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7284                        {
7285                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7286                                wait_secs = runtime.register_timeout.as_secs(),
7287                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7288                            active_child.spawn_failure = Some("registration_timeout".into());
7289                            if let Err(error) = stop_unregistered_child(active_child).await {
7290                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7291                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7292                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7293                                    &runtime.failure_context(true));
7294                                child = None;
7295                            }
7296                            // The ordinary exit handler records the child's exit,
7297                            // spends crash budget, and schedules recovery.
7298                            continue;
7299                        }
7300                    }
7301                    if health_probe.due() {
7302                        run_health_probe_cycle(
7303                            &spec,
7304                            &runtime,
7305                            &registry,
7306                            &process_liveness,
7307                            &snapshot,
7308                            &mut child,
7309                        ).await;
7310                        if child.is_none() {
7311                            runtime.start_requester = None;
7312                            runtime.pending_activity_restart = false;
7313                        }
7314                        if child.is_some() {
7315                            health_probe.schedule_next(&spec, runtime.health.cadence);
7316                        }
7317                    }
7318                }
7319            }
7320        } else if let Some(pending) = pending_respawn {
7321            tokio::select! {
7322                _ = runtime.child_roster.closed() => {
7323                    pending_respawn = None;
7324                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7325                        state.state = ModuleState::Stopped;
7326                        state.respawn_pending = false;
7327                    });
7328                    containment::terminate(&runtime.child_roster.containment).await;
7329                }
7330                _ = sleep_until(pending.deadline) => {
7331                    pending_respawn = None;
7332                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7333                    // A command handled below while the backoff elapsed may
7334                    // have stopped the module; never respawn past an operator's
7335                    // disable or drain.
7336                    if !respawn_still_pending(&snapshot) {
7337                        continue;
7338                    }
7339                    // The daemon began shutting down during the backoff: the
7340                    // spawn would be refused anyway, and refusing it here
7341                    // leaves the module stopped instead of reporting a
7342                    // failed restart.
7343                    if runtime.child_roster.is_closed() {
7344                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7345                            state.state = ModuleState::Stopped;
7346                        });
7347                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7348                        continue;
7349                    }
7350                    if let Err(err) = release_dead_registration(
7351                        &registry,
7352                        runtime.forwarding.as_deref(),
7353                        &snapshot,
7354                        &spec.module_id,
7355                        &runtime.failure_context(runtime.recovery_is_daemon),
7356                    ).await {
7357                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7358                        continue;
7359                    }
7360
7361                    if matches!(pending.kind, RespawnKind::Reload) {
7362                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7363                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7364                        if result.is_ok() && runtime.pending_activity_restart {
7365                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7366                        }
7367                        runtime.pending_activity_restart = false;
7368                        if let Some(reply) = reply { let _ = reply.send(result); }
7369                        continue;
7370                    }
7371                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7372                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7373                        Ok(next_child) => {
7374                            child = Some(next_child);
7375                            if runtime.pending_activity_restart {
7376                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7377                                runtime.pending_activity_restart = false;
7378                            }
7379                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7380                        }
7381                        Err(err) => {
7382                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7383                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7384                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7385                        }
7386                    }
7387                }
7388                command = commands.recv() => {
7389                    let Some(command) = command else {
7390                        containment::terminate(&runtime.child_roster.containment).await;
7391                        return;
7392                    };
7393                    if !handle_supervisor_command(
7394                        command,
7395                        &mut spec,
7396                        &mut runtime,
7397                        &registry,
7398                        &process_liveness,
7399                        &snapshot,
7400                        &mut child,
7401                        &mut commands,
7402                        &mut requeued,
7403                    ).await {
7404                        containment::terminate(&runtime.child_roster.containment).await;
7405                        return;
7406                    }
7407                    // Reconcile the pending respawn with what the command did:
7408                    // a start may already have spawned a fresh child,
7409                    // while a disable or drain moved the snapshot out of the
7410                    // state the respawn was counting down from.
7411                    if child.is_some() || !respawn_still_pending(&snapshot) {
7412                        pending_respawn = None;
7413                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7414                    }
7415                }
7416            }
7417        } else {
7418            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7419                (state.enabled && state.state == ModuleState::Failed)
7420                    .then_some(state.failed_retry.pending)
7421                    .flatten()
7422            });
7423            let command = if lock_snapshot(&snapshot)
7424                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7425            {
7426                tokio::select! {
7427                    command = commands.recv() => command,
7428                    _ = previous_instance_recheck.tick() => {
7429                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7430                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7431                            if enabled && !runtime.child_roster.is_closed() {
7432                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7433                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7434                                    Ok(next_child) => child = Some(next_child),
7435                                    Err(error) => {
7436                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7437                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7438                                    }
7439                                }
7440                            } else {
7441                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7442                                    state.waiting_for_previous_pid = None;
7443                                    state.waiting_for_previous_detail = None;
7444                                });
7445                            }
7446                        }
7447                        continue;
7448                    }
7449                }
7450            } else {
7451                tokio::select! {
7452                    biased;
7453                    command = commands.recv() => command,
7454                    _ = async {
7455                        match slow_retry {
7456                            Some(retry) => sleep_until(retry.deadline).await,
7457                            None => std::future::pending().await,
7458                        }
7459                    } => {
7460                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7461                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7462                            state.failed_retry.pending = None;
7463                        });
7464                        if runtime.child_roster.is_closed() {
7465                            continue;
7466                        }
7467                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7468                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7469                        if let Err(error) = set_child_enabled(
7470                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7471                            &runtime.failure_context(true),
7472                        ).await {
7473                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7474                        }
7475                        continue;
7476                    }
7477                }
7478            };
7479            let Some(command) = command else {
7480                containment::terminate(&runtime.child_roster.containment).await;
7481                return;
7482            };
7483            if !handle_supervisor_command(
7484                command,
7485                &mut spec,
7486                &mut runtime,
7487                &registry,
7488                &process_liveness,
7489                &snapshot,
7490                &mut child,
7491                &mut commands,
7492                &mut requeued,
7493            )
7494            .await
7495            {
7496                containment::terminate(&runtime.child_roster.containment).await;
7497                return;
7498            }
7499        }
7500    }
7501}
7502
7503// Preserve the deadline until the future is polled. Starting a relative sleep
7504// after the executor yields would restart part of the countdown.
7505async fn supervisor_sleep(wake_at: Option<Instant>) {
7506    match wake_at {
7507        Some(deadline) => sleep_until(deadline).await,
7508        None => std::future::pending().await,
7509    }
7510}
7511
7512fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7513    info!(
7514        module_id,
7515        restart_in_window = schedule.restart_in_window,
7516        delay_ms = schedule.delay.as_millis() as u64,
7517        "respawning after crash"
7518    );
7519}
7520
7521/// Whether the respawn a backoff was counting down to is still wanted. A
7522/// disable or drain handled while the backoff elapsed moves the snapshot out
7523/// of `Restarting`, and the operator's stop must win over the pending respawn,
7524/// so every sleep-then-spawn path re-validates against the live snapshot
7525/// instead of assuming the state it left behind still holds.
7526fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7527    matches!(
7528        lock_snapshot(snapshot),
7529        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7530    )
7531}
7532
7533enum NextAction {
7534    Stop {
7535        registration_released: bool,
7536    },
7537    Restart {
7538        schedule: Option<CrashRestartSchedule>,
7539    },
7540}
7541
7542#[allow(clippy::too_many_arguments)]
7543async fn handle_supervisor_command(
7544    command: SupervisorCommand,
7545    spec: &mut ModuleSpec,
7546    runtime: &mut SupervisorRuntimeConfig,
7547    registry: &Arc<Registry>,
7548    process_liveness: &SupervisorProcessLiveness,
7549    snapshot: &SharedSnapshot,
7550    child: &mut Option<SupervisedChild>,
7551    commands: &mut mpsc::Receiver<SupervisorCommand>,
7552    requeued: &mut VecDeque<SupervisorCommand>,
7553) -> bool {
7554    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7555        runtime.recovery_is_daemon = false;
7556    }
7557    if matches!(
7558        command,
7559        SupervisorCommand::Drain { .. }
7560            | SupervisorCommand::Retire { .. }
7561            | SupervisorCommand::SetEnabled { .. }
7562            | SupervisorCommand::Restart { .. }
7563    ) {
7564        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7565            state.failed_retry.cancel()
7566        });
7567    }
7568    let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7569        runtime
7570            .child_roster
7571            .previous_instance(&spec.module_id)
7572            .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7573    });
7574    let command = if let Some(error) = blocked {
7575        match command {
7576            SupervisorCommand::Restart { reply, .. }
7577            | SupervisorCommand::Reload { reply, .. }
7578            | SupervisorCommand::Swap { reply, .. } => {
7579                let _ = reply.send(Err(error));
7580                return true;
7581            }
7582            SupervisorCommand::SetEnabled {
7583                enabled: true,
7584                reply,
7585                ..
7586            } => {
7587                let _ = reply.send(Err(error));
7588                return true;
7589            }
7590            other => other,
7591        }
7592    } else {
7593        command
7594    };
7595    match command {
7596        SupervisorCommand::Drain { requester, reply } => {
7597            // A plain stop runs no forwarding drain, so nothing reaches the
7598            // module over its connection before the wait: ask by signal.
7599            let result = drain_optional_child(
7600                &spec.module_id,
7601                spec.protocol,
7602                StopNotice::NotSent,
7603                registry,
7604                runtime.forwarding.as_deref(),
7605                snapshot,
7606                &runtime.terminal_ring,
7607                &runtime.spawn_events,
7608                child,
7609                runtime.drain_timeout,
7610                ModuleState::Stopped,
7611                None,
7612                &FailureContext::operator(),
7613            )
7614            .await;
7615            containment::terminate(&runtime.child_roster.containment).await;
7616            let registration_released = result.is_ok();
7617            if registration_released {
7618                runtime.record_activity_transition(
7619                    "subc.module_stopped",
7620                    &spec.module_id,
7621                    requester,
7622                );
7623            }
7624            let _ = reply.send(result);
7625            if registration_released {
7626                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7627            }
7628            false
7629        }
7630        SupervisorCommand::Retire { requester, reply } => {
7631            let result = async {
7632                let stop_notice = begin_forwarding_drain_if_configured(
7633                    spec,
7634                    runtime,
7635                    registry,
7636                    snapshot,
7637                    None,
7638                    RouteCloseReason::Disable,
7639                )
7640                .await?;
7641                drain_optional_child(
7642                    &spec.module_id,
7643                    spec.protocol,
7644                    stop_notice,
7645                    registry,
7646                    runtime.forwarding.as_deref(),
7647                    snapshot,
7648                    &runtime.terminal_ring,
7649                    &runtime.spawn_events,
7650                    child,
7651                    runtime.drain_timeout,
7652                    ModuleState::Stopped,
7653                    None,
7654                    &FailureContext::operator(),
7655                )
7656                .await
7657            }
7658            .await;
7659            containment::terminate(&runtime.child_roster.containment).await;
7660            let registration_released = result.is_ok();
7661            if registration_released {
7662                runtime.record_activity_transition(
7663                    "subc.module_stopped",
7664                    &spec.module_id,
7665                    requester,
7666                );
7667            }
7668            let _ = reply.send(result);
7669            if registration_released {
7670                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7671            }
7672            false
7673        }
7674        SupervisorCommand::Restart {
7675            requester,
7676            drain_timeout_ms,
7677            received_at_generation,
7678            queued_at,
7679            reply,
7680        } => {
7681            // Without this line a restart that waited in the queue (behind a
7682            // health probe cycle or another command) was invisible: the log
7683            // showed only the drain timing out, minutes after the operator's call.
7684            info!(
7685                module_id = %spec.module_id,
7686                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7687                "restart command dequeued"
7688            );
7689            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7690            // caller whose own request lane rides the module being restarted: the
7691            // caller's in-flight request keeps the drain from quiescing, the drain
7692            // keeps the restart from completing, and the completion keeps the reply
7693            // from releasing the caller — so the drain always timed out and cut the
7694            // initiator with a GOODBYE, even on a healthy module. Replying once the
7695            // restart is validated lets a self-lane caller settle, which is exactly
7696            // what makes the drain succeed. Completion is observable via
7697            // supervisor.list / module status; a post-ack failure lands the module
7698            // in a visible terminal state below rather than in a reply nobody can
7699            // receive.
7700            let validation = match lock_snapshot(snapshot) {
7701                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7702                    module_id: spec.module_id.clone(),
7703                }),
7704                Ok(_) => Ok(()),
7705                Err(err) => Err(err),
7706            };
7707            let initiated = validation.is_ok();
7708            let _ = reply.send(validation);
7709            // A restart asks for a fresh process. Commands run one at a time,
7710            // so a restart queued behind another restart (two operator calls
7711            // in quick succession) is dequeued the moment the first one has
7712            // spawned its replacement -- before that process has sent HELLO.
7713            // Running it would drain and kill the process the first restart
7714            // just produced, which is the opposite of what both callers asked
7715            // for. If a process spawned after this request was received is
7716            // still supervised, the request is already satisfied. Not when the
7717            // configuration changed since that spawn: then the newer process
7718            // predates the spec this restart may exist to apply.
7719            // Generation publication can lag a child that is already serving
7720            // while its identity is recorded. Only process creation after request
7721            // receipt satisfies a restart, not publication after receipt.
7722            let satisfied_by_generation = if initiated
7723                && child
7724                    .as_ref()
7725                    .is_some_and(|child| child.spawned_at > queued_at)
7726            {
7727                lock_snapshot(snapshot).ok().and_then(|state| {
7728                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7729                })
7730            } else {
7731                None
7732            };
7733            let satisfied_by_pending = initiated
7734                && child.is_none()
7735                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7736                    // No child exists yet, so this replacement will necessarily
7737                    // be created after the queued request, unlike a published
7738                    // generation whose process may already have been serving.
7739                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7740                    if pending {
7741                        state.coalesced_restart_pending = true;
7742                    }
7743                    pending
7744                });
7745            if satisfied_by_pending {
7746                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7747            } else if let Some(generation) = satisfied_by_generation {
7748                info!(
7749                    module_id = %spec.module_id,
7750                    received_at_generation,
7751                    "restart already satisfied by generation {generation}; not restarting again"
7752                );
7753            } else if initiated {
7754                // Precedence: this restart's operator override, else the module's
7755                // configured budget (already resolved into the runtime).
7756                let drain_timeout = drain_timeout_ms
7757                    .map(Duration::from_millis)
7758                    .unwrap_or(runtime.drain_timeout);
7759                if let Err(err) = restart_child(
7760                    spec,
7761                    runtime,
7762                    registry,
7763                    process_liveness,
7764                    snapshot,
7765                    child,
7766                    drain_timeout,
7767                )
7768                .await
7769                {
7770                    warn!(
7771                        module_id = %spec.module_id,
7772                        error = %err,
7773                        "operator restart failed after initiation ack; module state carries the outcome"
7774                    );
7775                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7776                        state.state = ModuleState::Failed;
7777                        clear_current_process_facts(state);
7778                    });
7779                } else {
7780                    runtime.start_requester = requester;
7781                    runtime.pending_activity_restart = true;
7782                }
7783            }
7784            true
7785        }
7786        SupervisorCommand::Reload { requester, reply } => {
7787            let result =
7788                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7789            if result.is_ok() {
7790                runtime.start_requester = requester;
7791                runtime.pending_activity_restart = true;
7792            }
7793            if result.is_ok()
7794                && runtime
7795                    .scheduled_respawn
7796                    .lock()
7797                    .unwrap_or_else(|p| p.into_inner())
7798                    .as_ref()
7799                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7800            {
7801                *runtime
7802                    .deferred_reload_reply
7803                    .lock()
7804                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7805            } else {
7806                let _ = reply.send(result);
7807            }
7808            true
7809        }
7810        SupervisorCommand::SetEnabled {
7811            enabled,
7812            requester,
7813            reply,
7814        } => {
7815            let previous_requester = runtime.start_requester.clone();
7816            if enabled {
7817                runtime.start_requester = requester.clone();
7818            }
7819            let result = set_child_enabled(
7820                spec,
7821                runtime,
7822                registry,
7823                process_liveness,
7824                snapshot,
7825                child,
7826                enabled,
7827                &runtime.failure_context(false),
7828            )
7829            .await;
7830            if !enabled {
7831                containment::terminate(&runtime.child_roster.containment).await;
7832            }
7833            if matches!(result, Ok(true)) {
7834                runtime.pending_activity_restart = false;
7835                if enabled {
7836                    runtime.start_requester = requester;
7837                } else {
7838                    runtime.start_requester = None;
7839                    runtime.record_activity_transition(
7840                        "subc.module_disabled",
7841                        &spec.module_id,
7842                        requester,
7843                    );
7844                }
7845            } else {
7846                runtime.start_requester = previous_requester;
7847            }
7848            let _ = reply.send(result);
7849            true
7850        }
7851        SupervisorCommand::UpdateConfiguration {
7852            spec: next_spec,
7853            health,
7854            drain_timeout_ms,
7855            reply,
7856        } => {
7857            if let Some(handle) = &runtime.supervisor_handle {
7858                handle.apply_identity_configuration(&next_spec);
7859            }
7860            *spec = next_spec;
7861            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7862                state.configuration_updated_since_spawn = true;
7863            });
7864            let health_changed = runtime.health != health;
7865            runtime.health = health;
7866            // Reset the cadence and old endpoint's failure streak on a live
7867            // health-policy change rather than waiting for its old deadline.
7868            if health_changed {
7869                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7870                    state.health = ModuleHealthStatus::default();
7871                    state.last_probe_evidence = None;
7872                });
7873            }
7874            runtime.drain_timeout = drain_timeout_ms
7875                .map(Duration::from_millis)
7876                .unwrap_or(runtime.default_drain_timeout);
7877            *runtime
7878                .effective_drain_timeout
7879                .lock()
7880                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7881            let _ = reply.send(());
7882            true
7883        }
7884        SupervisorCommand::Swap {
7885            requester,
7886            ready_timeout,
7887            reply,
7888        } => {
7889            if spec.carry_containment
7890                || containment::current(&runtime.child_roster.containment).is_some()
7891            {
7892                let _ = reply.send(Err(SuperviseError::InvalidSpec {
7893                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
7894                }));
7895                return true;
7896            }
7897            let mut swap_runtime = runtime.clone();
7898            swap_runtime.start_requester = requester;
7899            let before = lock_snapshot(snapshot)
7900                .ok()
7901                .map(|state| state.spawn_generation);
7902            let end = swap::run_swap(
7903                spec,
7904                &swap_runtime,
7905                registry,
7906                process_liveness,
7907                snapshot,
7908                child,
7909                commands,
7910                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7911                reply,
7912            )
7913            .await;
7914            if before
7915                != lock_snapshot(snapshot)
7916                    .ok()
7917                    .map(|state| state.spawn_generation)
7918            {
7919                runtime.start_requester = swap_runtime.start_requester;
7920            }
7921            requeued.extend(end.requeue);
7922            true
7923        }
7924    }
7925}
7926
7927async fn restart_child(
7928    spec: &ModuleSpec,
7929    runtime: &SupervisorRuntimeConfig,
7930    registry: &Registry,
7931    process_liveness: &SupervisorProcessLiveness,
7932    snapshot: &SharedSnapshot,
7933    child: &mut Option<SupervisedChild>,
7934    drain_timeout: Duration,
7935) -> Result<(), SuperviseError> {
7936    // Restart cycles a running module; it must not silently start a disabled one.
7937    if !lock_snapshot(snapshot)?.enabled {
7938        return Err(SuperviseError::Disabled {
7939            module_id: spec.module_id.clone(),
7940        });
7941    }
7942    let stop_notice = begin_forwarding_drain_with_timeout(
7943        spec,
7944        runtime,
7945        registry,
7946        snapshot,
7947        None,
7948        RouteCloseReason::Restart,
7949        drain_timeout,
7950    )
7951    .await?;
7952
7953    if child.is_some() {
7954        drain_optional_child(
7955            &spec.module_id,
7956            spec.protocol,
7957            stop_notice,
7958            registry,
7959            runtime.forwarding.as_deref(),
7960            snapshot,
7961            &runtime.terminal_ring,
7962            &runtime.spawn_events,
7963            child,
7964            drain_timeout,
7965            ModuleState::Restarting,
7966            Some(true),
7967            &FailureContext::operator(),
7968        )
7969        .await?;
7970    } else {
7971        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7972            state.enabled = true;
7973            state.state = ModuleState::Restarting;
7974            clear_current_process_facts(state);
7975        })?;
7976        release_dead_registration(
7977            registry,
7978            runtime.forwarding.as_deref(),
7979            snapshot,
7980            &spec.module_id,
7981            &runtime.failure_context(false),
7982        )
7983        .await?;
7984    }
7985
7986    reset_restart_count(snapshot, &spec.module_id)?;
7987    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7988    schedule_respawn(
7989        runtime,
7990        snapshot,
7991        &spec.module_id,
7992        runtime.restart_policy.backoff,
7993        RespawnKind::Spawn,
7994        false,
7995    )
7996}
7997
7998async fn reload_child(
7999    spec: &ModuleSpec,
8000    runtime: &SupervisorRuntimeConfig,
8001    registry: &Registry,
8002    process_liveness: &SupervisorProcessLiveness,
8003    snapshot: &SharedSnapshot,
8004    child: &mut Option<SupervisedChild>,
8005) -> Result<(), SuperviseError> {
8006    // Reload cycles a running module; it must not silently start a disabled one.
8007    if !lock_snapshot(snapshot)?.enabled {
8008        return Err(SuperviseError::Disabled {
8009            module_id: spec.module_id.clone(),
8010        });
8011    }
8012    let stop_notice = begin_forwarding_drain(
8013        spec,
8014        runtime,
8015        registry,
8016        snapshot,
8017        Some(true),
8018        RouteCloseReason::Reload,
8019    )
8020    .await?;
8021
8022    if child.is_some() {
8023        drain_optional_child(
8024            &spec.module_id,
8025            spec.protocol,
8026            stop_notice,
8027            registry,
8028            runtime.forwarding.as_deref(),
8029            snapshot,
8030            &runtime.terminal_ring,
8031            &runtime.spawn_events,
8032            child,
8033            runtime.drain_timeout,
8034            ModuleState::Restarting,
8035            Some(true),
8036            &FailureContext::operator(),
8037        )
8038        .await?;
8039    } else {
8040        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8041            state.enabled = true;
8042            state.state = ModuleState::Restarting;
8043            clear_current_process_facts(state);
8044        })?;
8045        release_dead_registration(
8046            registry,
8047            runtime.forwarding.as_deref(),
8048            snapshot,
8049            &spec.module_id,
8050            &runtime.failure_context(false),
8051        )
8052        .await?;
8053    }
8054
8055    reset_restart_count(snapshot, &spec.module_id)?;
8056    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8057    schedule_respawn(
8058        runtime,
8059        snapshot,
8060        &spec.module_id,
8061        runtime.restart_policy.backoff,
8062        RespawnKind::Reload,
8063        false,
8064    )
8065}
8066
8067async fn finish_reload_child(
8068    spec: &ModuleSpec,
8069    runtime: &SupervisorRuntimeConfig,
8070    registry: &Registry,
8071    process_liveness: &SupervisorProcessLiveness,
8072    snapshot: &SharedSnapshot,
8073    child: &mut Option<SupervisedChild>,
8074) -> Result<(), SuperviseError> {
8075    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8076    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8077        Ok(next_child) => next_child,
8078        Err(err) => {
8079            return handle_reload_spawn_failure(
8080                spec,
8081                runtime,
8082                process_liveness,
8083                snapshot,
8084                child,
8085                format!("new child failed to spawn: {err}"),
8086            )
8087            .await;
8088        }
8089    };
8090    *child = Some(next_child);
8091
8092    let wait_outcome = {
8093        let active_child = child.as_mut().expect("new reload child was just stored");
8094        wait_for_registration_after_reload(
8095            registry,
8096            &spec.module_id,
8097            snapshot,
8098            active_child,
8099            REGISTRY_RELEASE_TIMEOUT,
8100        )
8101        .await?
8102    };
8103
8104    match wait_outcome {
8105        RegistrationWaitOutcome::Registered => {
8106            child
8107                .as_mut()
8108                .expect("registered reload child")
8109                .registration_deadline = None;
8110            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8111            Ok(())
8112        }
8113        RegistrationWaitOutcome::Exited(exit_report) => {
8114            if let Some(active_child) = child.as_mut() {
8115                active_child.drain_stderr(&spec.module_id).await;
8116            }
8117            // Keep the exited child's roster guard until its terminal is written.
8118            // Shutdown waits on that guard, not on the child Option used for respawn.
8119            let mut exited_child = child.take().expect("exited reload child is still stored");
8120            #[cfg(test)]
8121            if let Some(gate) = &runtime.test_reload_exit_record_gate {
8122                gate.reached.notify_one();
8123                gate.resume.notified().await;
8124            }
8125            let result = handle_reload_child_registration_failure(
8126                spec,
8127                runtime,
8128                registry,
8129                process_liveness,
8130                snapshot,
8131                child,
8132                ReloadRegistrationFailure {
8133                    exit_report: registration_failure_exit_report(exit_report),
8134                    reason: exited_child
8135                        .spawn_failure
8136                        .clone()
8137                        .unwrap_or_else(|| "new child exited before registering".to_string()),
8138                },
8139            )
8140            .await;
8141            exited_child
8142                .finish_exit(false)
8143                .await
8144                .map_err(|source| SuperviseError::Wait {
8145                    module_id: spec.module_id.clone(),
8146                    source,
8147                })?;
8148            result
8149        }
8150        RegistrationWaitOutcome::TimedOut => {
8151            let mut timed_out_child = child
8152                .take()
8153                .expect("timed-out reload child is still running");
8154            timed_out_child
8155                .start_kill()
8156                .map_err(|source| SuperviseError::Kill {
8157                    module_id: spec.module_id.clone(),
8158                    source,
8159                })?;
8160            let status = timed_out_child
8161                .wait()
8162                .await
8163                .map_err(|source| SuperviseError::Wait {
8164                    module_id: spec.module_id.clone(),
8165                    source,
8166                })?;
8167            timed_out_child.drain_stderr(&spec.module_id).await;
8168            let result = handle_reload_child_registration_failure(
8169                spec,
8170                runtime,
8171                registry,
8172                process_liveness,
8173                snapshot,
8174                child,
8175                ReloadRegistrationFailure {
8176                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8177                        snapshot,
8178                        &timed_out_child,
8179                        &status,
8180                    )),
8181                    reason: format!(
8182                        "new child did not register within {:?}",
8183                        REGISTRY_RELEASE_TIMEOUT
8184                    ),
8185                },
8186            )
8187            .await;
8188            timed_out_child
8189                .finish_exit(false)
8190                .await
8191                .map_err(|source| SuperviseError::Wait {
8192                    module_id: spec.module_id.clone(),
8193                    source,
8194                })?;
8195            result
8196        }
8197    }
8198}
8199
8200#[allow(clippy::too_many_arguments)]
8201async fn set_child_enabled(
8202    spec: &ModuleSpec,
8203    runtime: &SupervisorRuntimeConfig,
8204    registry: &Registry,
8205    process_liveness: &SupervisorProcessLiveness,
8206    snapshot: &SharedSnapshot,
8207    child: &mut Option<SupervisedChild>,
8208    enabled: bool,
8209    failure: &FailureContext,
8210) -> Result<bool, SuperviseError> {
8211    let (current_enabled, current_state, respawn_pending) = {
8212        let state = lock_snapshot(snapshot)?;
8213        (state.enabled, state.state, state.respawn_pending)
8214    };
8215    // `start` (enable on an already-enabled module) heals TERMINAL states instead
8216    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
8217    // clean (Stopped) has no live process and no other in-band recovery — the
8218    // operator's start is the explicit recovery act and resets the budget. Without
8219    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
8220    // which the 2026-07-14 aft outage proved is a trap when the failed module is
8221    // the one providing every agent's shell.
8222    let revive_terminal = enabled
8223        && current_enabled
8224        && child.is_none()
8225        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8226            || (current_state == ModuleState::Restarting && !respawn_pending));
8227    if current_enabled == enabled && !revive_terminal {
8228        return Ok(false);
8229    }
8230
8231    if enabled {
8232        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8233            state.enabled = true;
8234            state.state = ModuleState::Starting;
8235            clear_current_process_facts(state);
8236        })?;
8237        #[cfg(test)]
8238        if runtime.test_seed_stale_facts_before_enable_spawn {
8239            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8240                state.process_alive = true;
8241                state.pid = Some(41);
8242                state.spawned_at_ms = Some(42);
8243                state.spawned_from = Some(PathBuf::from("/spawned/module"));
8244                state.spawned_file_identity = Some(SpawnedFileIdentity {
8245                    device: 43,
8246                    inode: 44,
8247                    #[cfg(windows)]
8248                    windows_image: None,
8249                });
8250            })?;
8251        }
8252        release_dead_registration(
8253            registry,
8254            runtime.forwarding.as_deref(),
8255            snapshot,
8256            &spec.module_id,
8257            failure,
8258        )
8259        .await?;
8260        reset_restart_count(snapshot, &spec.module_id)?;
8261        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8262        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8263            Ok(next_child) => next_child,
8264            Err(err) => {
8265                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8266                process_liveness.untrack_if_current(&spec.module_id, snapshot);
8267                return Err(err);
8268            }
8269        };
8270        *child = Some(next_child);
8271        debug!(module_id = %spec.module_id, "supervised module enabled");
8272        Ok(true)
8273    } else {
8274        let stop_notice = begin_forwarding_drain_if_configured(
8275            spec,
8276            runtime,
8277            registry,
8278            snapshot,
8279            Some(false),
8280            RouteCloseReason::Disable,
8281        )
8282        .await?;
8283        drain_optional_child(
8284            &spec.module_id,
8285            spec.protocol,
8286            stop_notice,
8287            registry,
8288            runtime.forwarding.as_deref(),
8289            snapshot,
8290            &runtime.terminal_ring,
8291            &runtime.spawn_events,
8292            child,
8293            runtime.drain_timeout,
8294            ModuleState::Disabled,
8295            Some(false),
8296            failure,
8297        )
8298        .await?;
8299        debug!(module_id = %spec.module_id, "supervised module disabled");
8300        Ok(true)
8301    }
8302}
8303
8304#[allow(clippy::too_many_arguments)]
8305async fn on_child_exit(
8306    spec: &ModuleSpec,
8307    policy: RestartPolicy,
8308    registry: &Registry,
8309    snapshot: &SharedSnapshot,
8310    terminal_ring: &Arc<Mutex<TerminalRing>>,
8311    spawn_events: &SpawnEventFeed,
8312    roster: &ChildRoster,
8313    exit_report: ExitReport,
8314    daemon_initiated: bool,
8315) -> NextAction {
8316    // Once the daemon has begun shutting down, no exit is a crash to recover
8317    // from: the module is exiting because the daemon is going away (EOF on its
8318    // connection, or a service manager signalling the whole cgroup). Record it
8319    // as such and never schedule a respawn, which would only start a process
8320    // for the shutdown to end again.
8321    if roster.is_closed() {
8322        return on_child_exit_during_daemon_shutdown(
8323            spec,
8324            registry,
8325            snapshot,
8326            terminal_ring,
8327            spawn_events,
8328            exit_report,
8329        )
8330        .await;
8331    }
8332    // Deliberate stops the supervisor asks for (operator stop, disable,
8333    // restart, reload, swap, a health restart, a drain that runs out of budget)
8334    // takes the child out of the supervise loop and reaps it in
8335    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8336    // that reaches this point is either unrequested or a first-HELLO timeout,
8337    // which is deliberately classified as a crash even after a clean TERM exit.
8338    //
8339    // For a subc-wire module a clean exit is still a stop: those modules are
8340    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8341    // crash, and exiting 0 is a deliberate choice the module made. A
8342    // `protocol: "none"` module is a stock program we cannot change, and many
8343    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8344    // stop would leave the module down for good after any stray signal, so it
8345    // goes through the crash path instead: it spends restart budget, respawns
8346    // with the crash backoff, and ends `failed` when the budget runs out.
8347    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8348        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8349    match exit_report.kind {
8350        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8351            info!(
8352                module_id = %spec.module_id,
8353                exit_code = ?exit_report.code,
8354                exit_signal = ?exit_report.signal,
8355                "supervised module exited cleanly"
8356            );
8357            if let Err(err) = publish_after_terminal_record(
8358                &spec.module_id,
8359                || {
8360                    record_terminal(
8361                        &spec.module_id,
8362                        terminal_ring,
8363                        spawn_events,
8364                        &exit_report,
8365                        TerminalDisposition::Stopped,
8366                    )
8367                },
8368                || {
8369                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8370                        state.state = ModuleState::Stopped;
8371                        clear_current_process_facts(state);
8372                        state.last_exit = Some(exit_report.clone());
8373                    })
8374                },
8375            ) {
8376                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8377            }
8378            let registration_released = match wait_for_registration_release(
8379                registry,
8380                &spec.module_id,
8381                REGISTRY_RELEASE_TIMEOUT,
8382            )
8383            .await
8384            {
8385                Ok(()) => true,
8386                Err(err) => {
8387                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8388                    false
8389                }
8390            };
8391            NextAction::Stop {
8392                registration_released,
8393            }
8394        }
8395        ExitKind::Clean | ExitKind::Crash => {
8396            if unrequested_clean_exit_of_protocol_none {
8397                warn!(
8398                    module_id = %spec.module_id,
8399                    exit_code = ?exit_report.code,
8400                    exit_signal = ?exit_report.signal,
8401                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8402                );
8403            } else {
8404                warn!(
8405                    module_id = %spec.module_id,
8406                    exit_code = ?exit_report.code,
8407                    exit_signal = ?exit_report.signal,
8408                    "supervised module exited abnormally (crash)"
8409                );
8410            }
8411            let mut restart_schedule = None;
8412            let mut disposition = TerminalDisposition::Disabled;
8413            let mut final_state = ModuleState::Disabled;
8414            // Set only when the budget is what stopped the module, so the
8415            // terminal record says which limit was hit rather than leaving
8416            // `failed` to be read as "crashed once, badly".
8417            let mut disposition_detail = lock_snapshot(snapshot)
8418                .ok()
8419                .and_then(|mut state| state.spawn_failure.take());
8420            let now = Instant::now();
8421            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8422                if state.enabled {
8423                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8424                        // Reserving the last slot exhausts the budget for the
8425                        // replacement, not for this exit. Publish its scheduled
8426                        // recovery atomically with the reservation so connection
8427                        // cleanup sees it while terminal recording still holds
8428                        // the visible state at Running.
8429                        state.respawn_pending = true;
8430                        final_state = ModuleState::Restarting;
8431                        restart_schedule = Some(schedule);
8432                        disposition = TerminalDisposition::Restarting;
8433                    } else {
8434                        final_state = ModuleState::Failed;
8435                        disposition = TerminalDisposition::Failed;
8436                        let budget = policy.budget_exhausted_detail();
8437                        disposition_detail =
8438                            Some(disposition_detail.take().map_or_else(
8439                                || budget.clone(),
8440                                |cause| format!("{cause}; {budget}"),
8441                            ));
8442                    }
8443                } else {
8444                    disposition = TerminalDisposition::Disabled;
8445                }
8446            }) {
8447                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8448                return NextAction::Stop {
8449                    registration_released: false,
8450                };
8451            }
8452            if disposition == TerminalDisposition::Failed {
8453                // The window is in the message, not only in the fields: this line
8454                // is read in a scrollback where a bare `max_restarts=3` reads as a
8455                // lifetime cap and sends the operator looking for three crashes
8456                // that never happened together.
8457                error!(
8458                    module_id = %spec.module_id,
8459                    max_restarts = policy.max_restarts,
8460                    window_secs = policy.window.as_secs(),
8461                    "module stopped: {}",
8462                    policy.budget_exhausted_detail()
8463                );
8464            }
8465            if let Err(err) = publish_after_terminal_record(
8466                &spec.module_id,
8467                || {
8468                    record_terminal_with_detail(
8469                        &spec.module_id,
8470                        terminal_ring,
8471                        spawn_events,
8472                        &exit_report,
8473                        disposition,
8474                        disposition_detail,
8475                    )
8476                },
8477                || {
8478                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8479                        state.state = final_state;
8480                        clear_current_process_facts(state);
8481                        if final_state == ModuleState::Failed
8482                            && daemon_initiated
8483                            && !roster.is_closed()
8484                        {
8485                            state.failed_retry.schedule(&spec.module_id);
8486                        }
8487                        state.last_exit = Some(exit_report.clone());
8488                    })
8489                },
8490            ) {
8491                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8492            }
8493
8494            if let Some(schedule) = restart_schedule {
8495                NextAction::Restart {
8496                    schedule: Some(schedule),
8497                }
8498            } else {
8499                let registration_released = match wait_for_registration_release(
8500                    registry,
8501                    &spec.module_id,
8502                    REGISTRY_RELEASE_TIMEOUT,
8503                )
8504                .await
8505                {
8506                    Ok(()) => true,
8507                    Err(err) => {
8508                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8509                        false
8510                    }
8511                };
8512                NextAction::Stop {
8513                    registration_released,
8514                }
8515            }
8516        }
8517        ExitKind::DeliberateSeverance => {
8518            warn!(
8519                module_id = %spec.module_id,
8520                exit_code = ?exit_report.code,
8521                exit_signal = ?exit_report.signal,
8522                "supervised module exited after deliberate connection severance"
8523            );
8524            let should_restart = match lock_snapshot(snapshot) {
8525                Ok(state) => state.enabled,
8526                Err(err) => {
8527                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8528                    return NextAction::Stop {
8529                        registration_released: false,
8530                    };
8531                }
8532            };
8533            let final_state = if should_restart {
8534                ModuleState::Restarting
8535            } else {
8536                ModuleState::Disabled
8537            };
8538            if let Err(err) = publish_after_terminal_record(
8539                &spec.module_id,
8540                || {
8541                    record_terminal(
8542                        &spec.module_id,
8543                        terminal_ring,
8544                        spawn_events,
8545                        &exit_report,
8546                        terminal_disposition(final_state),
8547                    )
8548                },
8549                || {
8550                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8551                        state.state = final_state;
8552                        clear_current_process_facts(state);
8553                        state.last_exit = Some(exit_report.clone());
8554                        state.lifetime_restarts += 1;
8555                    })
8556                },
8557            ) {
8558                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8559                return NextAction::Stop {
8560                    registration_released: false,
8561                };
8562            }
8563
8564            if should_restart {
8565                NextAction::Restart { schedule: None }
8566            } else {
8567                let registration_released = match wait_for_registration_release(
8568                    registry,
8569                    &spec.module_id,
8570                    REGISTRY_RELEASE_TIMEOUT,
8571                )
8572                .await
8573                {
8574                    Ok(()) => true,
8575                    Err(err) => {
8576                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8577                        false
8578                    }
8579                };
8580                NextAction::Stop {
8581                    registration_released,
8582                }
8583            }
8584        }
8585    }
8586}
8587
8588async fn on_child_exit_during_daemon_shutdown(
8589    spec: &ModuleSpec,
8590    registry: &Registry,
8591    snapshot: &SharedSnapshot,
8592    terminal_ring: &Arc<Mutex<TerminalRing>>,
8593    spawn_events: &SpawnEventFeed,
8594    exit_report: ExitReport,
8595) -> NextAction {
8596    info!(
8597        module_id = %spec.module_id,
8598        exit_code = ?exit_report.code,
8599        exit_signal = ?exit_report.signal,
8600        exit_kind = ?exit_report.kind,
8601        "supervised module exited during daemon shutdown; not restarting it"
8602    );
8603    if let Err(err) = publish_after_terminal_record(
8604        &spec.module_id,
8605        || {
8606            record_terminal(
8607                &spec.module_id,
8608                terminal_ring,
8609                spawn_events,
8610                &exit_report,
8611                TerminalDisposition::DaemonShutdown,
8612            )
8613        },
8614        || {
8615            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8616                state.state = ModuleState::Stopped;
8617                clear_current_process_facts(state);
8618                state.last_exit = Some(exit_report.clone());
8619            })
8620        },
8621    ) {
8622        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8623    }
8624    let registration_released =
8625        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8626            .await
8627            .is_ok();
8628    NextAction::Stop {
8629        registration_released,
8630    }
8631}
8632
8633// Keep the status lock out of terminal I/O. Readers may observe the previous
8634// state while recording is in progress, but never a completed exit without its
8635// history. Journal errors are logged by the journal and retained in the ring;
8636// a recording panic still publishes the final state before resuming the unwind.
8637fn publish_after_terminal_record<T>(
8638    module_id: &str,
8639    record: impl FnOnce(),
8640    publish: impl FnOnce() -> T,
8641) -> T {
8642    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8643    let published = publish();
8644    if let Err(panic) = recorded {
8645        error!(
8646            module_id,
8647            "terminal recording panicked; published final state before resuming unwind"
8648        );
8649        std::panic::resume_unwind(panic);
8650    }
8651    published
8652}
8653
8654fn fail_after_wait_error(
8655    module_id: &str,
8656    snapshot: &SharedSnapshot,
8657    terminal_ring: &Arc<Mutex<TerminalRing>>,
8658    spawn_events: &SpawnEventFeed,
8659    detail: String,
8660    failure: &FailureContext,
8661) {
8662    let report = wait_error_exit_report();
8663    publish_after_terminal_record(
8664        module_id,
8665        || {
8666            record_terminal_with_detail(
8667                module_id,
8668                terminal_ring,
8669                spawn_events,
8670                &report,
8671                TerminalDisposition::Failed,
8672                Some(detail),
8673            )
8674        },
8675        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8676    );
8677}
8678
8679#[cfg(test)]
8680fn record_wait_error_terminal(
8681    module_id: &str,
8682    terminal_ring: &Arc<Mutex<TerminalRing>>,
8683    spawn_events: &SpawnEventFeed,
8684) {
8685    record_terminal(
8686        module_id,
8687        terminal_ring,
8688        spawn_events,
8689        &wait_error_exit_report(),
8690        TerminalDisposition::Failed,
8691    );
8692}
8693
8694fn record_terminal(
8695    module_id: &str,
8696    terminal_ring: &Arc<Mutex<TerminalRing>>,
8697    spawn_events: &SpawnEventFeed,
8698    exit_report: &ExitReport,
8699    disposition: TerminalDisposition,
8700) {
8701    record_terminal_with_detail(
8702        module_id,
8703        terminal_ring,
8704        spawn_events,
8705        exit_report,
8706        disposition,
8707        None,
8708    );
8709}
8710
8711/// The ring lock is held only to capture the read (see
8712/// `TerminalJournal::capture_read`), so this module's exits keep recording
8713/// while the journal files are read. Blocking: it reads files.
8714fn durable_terminal_history_of(
8715    terminal_ring: &Mutex<TerminalRing>,
8716    module_id: &str,
8717) -> subc_control::TerminalHistory {
8718    let read = terminal_ring
8719        .lock()
8720        .unwrap_or_else(|p| p.into_inner())
8721        .capture_durable_history();
8722    read.read(module_id)
8723}
8724
8725fn record_terminal_with_detail(
8726    module_id: &str,
8727    terminal_ring: &Arc<Mutex<TerminalRing>>,
8728    spawn_events: &SpawnEventFeed,
8729    exit_report: &ExitReport,
8730    disposition: TerminalDisposition,
8731    disposition_detail: Option<String>,
8732) {
8733    #[cfg(test)]
8734    terminal_history_tests::before_terminal_record(spawn_events);
8735    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8736    let record = TerminalRecord {
8737        exit_code: exit_report.code,
8738        exit_signal: exit_report.signal,
8739        at_ms: exit_report.at_ms,
8740        disposition,
8741        exit_kind: exit_report.kind.into(),
8742        disposition_detail,
8743    };
8744    terminal_ring
8745        .lock()
8746        .unwrap_or_else(|poisoned| poisoned.into_inner())
8747        .record_exit(module_id, record);
8748}
8749
8750fn untrack_if_registration_released(
8751    process_liveness: &SupervisorProcessLiveness,
8752    registry: &Registry,
8753    module_id: &str,
8754    snapshot: &SharedSnapshot,
8755) {
8756    match registry.get_module(module_id) {
8757        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8758        Ok(Some(_)) => {}
8759        Err(err) => {
8760            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8761        }
8762    }
8763}
8764
8765/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8766/// then apply the module's configured entries minus daemon-private capture keys.
8767///
8768/// Separated from `spawn_child` only so it can be asserted without spawning a
8769/// process — a duplicate of this logic in a test would pass while the real one
8770/// drifted, which is the defect class this function exists to avoid.
8771/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8772/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8773/// either and the argument would stop a stock binary from starting at all.
8774/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8775///
8776/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8777/// through [`apply_wire_spawn_args_for_role`].
8778#[cfg(test)]
8779fn apply_wire_spawn_args(
8780    command: &mut Command,
8781    spec: &ModuleSpec,
8782    connection_file_path: Option<&std::path::Path>,
8783    handle: Option<&SupervisorHandle>,
8784) -> Result<Option<NonceHandoff>, SuperviseError> {
8785    apply_wire_spawn_args_for_role(
8786        command,
8787        spec,
8788        connection_file_path,
8789        handle,
8790        SpawnRole::Plain,
8791    )
8792}
8793
8794/// The read end of a spawn's launch-nonce pipe, prepared by
8795/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8796/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8797/// server before spawn and authenticates its client after the child exists.
8798#[cfg(unix)]
8799type NonceHandoff = subc_os::LaunchNonceHandoff;
8800#[cfg(windows)]
8801type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8802#[cfg(not(any(unix, windows)))]
8803type NonceHandoff = std::convert::Infallible;
8804
8805/// Create the launch nonce that a new module process must present when it
8806/// registers or opens a consumer connection, including a replacement process
8807/// started alongside a still-serving module.
8808///
8809/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8810/// a separate candidate token so the still-serving incumbent and its consumers
8811/// keep their nonce. Both records are installed before the process exists, so
8812/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8813///
8814/// On Unix the nonce is delivered only through a pipe. It is written into
8815/// a pipe whose read end the child gets as descriptor 3, named by
8816/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8817/// process of the same user cannot read it with `ps eww`. That handoff is
8818/// returned rather than installed here, because installing it replaces
8819/// whatever the child has at descriptor 3 and so must be the last pre-exec
8820/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8821/// the Linux callback that writes the process into its cgroup. Only after that
8822/// does `handoff.install_last` register the descriptor-3 replacement, so it
8823/// cannot close a descriptor the cgroup callback still needs.
8824/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8825fn apply_wire_spawn_args_for_role(
8826    command: &mut Command,
8827    spec: &ModuleSpec,
8828    connection_file_path: Option<&std::path::Path>,
8829    handle: Option<&SupervisorHandle>,
8830    role: SpawnRole,
8831) -> Result<Option<NonceHandoff>, SuperviseError> {
8832    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8833    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8834    // included: a daemon started from a module's process tree inherits it,
8835    // and passing it on would point the child at a descriptor it does not
8836    // have.
8837    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8838    #[cfg(windows)]
8839    {
8840        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8841        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8842    }
8843    // Remove inherited or configured copies too: withholding must mean absent.
8844    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8845    if spec.protocol == ModuleProtocol::None {
8846        return Ok(None);
8847    }
8848    if let Some(connection_file_path) = connection_file_path {
8849        command.arg(SUBC_ARG).arg(connection_file_path);
8850    }
8851
8852    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8853    // route.open attestation. Reserved modules additionally use the same nonce
8854    // for HELLO id-squatting protection. A respawn rotates both records.
8855    let nonce = generate_launch_nonce()?;
8856    if let Some(handle) = handle {
8857        match role {
8858            SpawnRole::Plain => {
8859                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8860                if spec.reserved {
8861                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8862                }
8863            }
8864            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8865        }
8866    }
8867    #[cfg(unix)]
8868    let handoff = {
8869        let handoff =
8870            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8871                program: spec.program.clone(),
8872                source,
8873                cgroup_path: None,
8874            })?;
8875        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8876        Some(handoff)
8877    };
8878    #[cfg(windows)]
8879    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8880    #[cfg(not(any(unix, windows)))]
8881    let handoff = None;
8882    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8883    // until the daemon's per-spawn delivery records show that every running
8884    // Windows module has read its secret from the named pipe.
8885    #[cfg(not(unix))]
8886    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8887    Ok(handoff)
8888}
8889
8890fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8891    command.env_remove(CK_LOG_ENV);
8892    // The spawn role is the supervisor's to set, and only on a swap candidate
8893    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8894    // it, is what makes it absent on a plain spawn: the daemon's own
8895    // environment could carry it, and so could a spec built outside daemon
8896    // config (config refuses it as an `env` key). A module reading it on a
8897    // plain restart would pick the long swap budget and leave callers waiting.
8898    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8899    for (key, value) in &spec.env {
8900        // cortexkit-log currently exposes retention only as a Rust struct, not
8901        // environment names. These values are daemon-private sink metadata and
8902        // must never become a public child-process contract by being inherited.
8903        if matches!(
8904            key.as_str(),
8905            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8906        ) || key == SUBC_SPAWN_ROLE_ENV
8907        {
8908            continue;
8909        }
8910        command.env(key, value);
8911    }
8912}
8913
8914#[cfg(windows)]
8915fn prepare_windows_nonce_pipe(
8916    command: &mut Command,
8917    spec: &ModuleSpec,
8918    nonce: &str,
8919) -> Option<NonceHandoff> {
8920    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8921        Ok(handoff) => {
8922            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8923            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8924            // still register until every module reads the pipe. A reader without
8925            // this variable set to env returns the pipe error instead of using
8926            // an inherited or configured environment copy.
8927            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8928            Some(handoff)
8929        }
8930        Err(error) => {
8931            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8932            None
8933        }
8934    }
8935}
8936
8937#[cfg(windows)]
8938fn start_windows_nonce_pipe(
8939    handoff: Option<NonceHandoff>,
8940    pid: u32,
8941    spec: &ModuleSpec,
8942) -> Option<subc_os::LaunchNoncePipeDelivery> {
8943    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8944    // same maximum wait used for registration during a reload. Older modules
8945    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8946    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8947        Ok(delivery) => Some(delivery),
8948        Err(error) => {
8949            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8950            None
8951        }
8952    })
8953}
8954
8955fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8956    #[cfg(windows)]
8957    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8958        return Some(
8959            if snapshot
8960                .pipe_consumed
8961                .as_ref()
8962                .is_some_and(|consumed| consumed.get().is_some())
8963            {
8964                "pipe"
8965            } else {
8966                "env"
8967            }
8968            .to_owned(),
8969        );
8970    }
8971    let _ = snapshot;
8972    None
8973}
8974
8975/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8976/// of a blue/green swap.
8977#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8978enum SpawnRole {
8979    Plain,
8980    SwapCandidate,
8981}
8982
8983/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8984/// `apply_child_env` has already removed the variable for every spawn.
8985fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8986    if role == SpawnRole::SwapCandidate {
8987        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8988    }
8989}
8990
8991fn previous_instance_error(
8992    spec: &ModuleSpec,
8993    pid: u32,
8994    reason: crate::live_children::RetainedReason,
8995) -> SuperviseError {
8996    SuperviseError::Spawn {
8997        program: spec.program.clone(),
8998        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8999        cgroup_path: None,
9000    }
9001}
9002
9003fn carries_containment(configured: bool, has_container: bool) -> bool {
9004    configured || has_container
9005}
9006
9007fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
9008    if !carries_containment(
9009        spec.carry_containment,
9010        containment::current(&roster.containment).is_some(),
9011    ) {
9012        return None;
9013    }
9014    Some(SuperviseError::Spawn {
9015        program: spec.program.clone(),
9016        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9017        cgroup_path: None,
9018    })
9019}
9020
9021fn spawn_child(
9022    spec: &ModuleSpec,
9023    connection_file_path: Option<&std::path::Path>,
9024    handle: Option<&SupervisorHandle>,
9025    ring: &Arc<Mutex<StderrRing>>,
9026    capture_logs_dir: Option<&std::path::Path>,
9027    roster: &ChildRoster,
9028    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9029) -> Result<SupervisedChild, SuperviseError> {
9030    #[cfg(all(test, unix))]
9031    if let Some(source) = containment_tests::join_failure(spec, roster) {
9032        containment::terminate_blocking(&roster.containment);
9033        return Err(SuperviseError::Spawn {
9034            program: spec.program.clone(),
9035            source,
9036            cgroup_path: None,
9037        });
9038    }
9039    let result = spawn_child_in_slot(
9040        spec,
9041        connection_file_path,
9042        handle,
9043        ring,
9044        capture_logs_dir,
9045        roster,
9046        #[cfg(target_os = "linux")]
9047        cgroup_placement,
9048        SpawnRole::Plain,
9049        false,
9050    );
9051    if result.is_err() {
9052        if roster.admission_pending(&spec.module_id) {
9053            containment::terminate_deferred(&roster.containment);
9054        } else {
9055            containment::terminate_blocking(&roster.containment);
9056        }
9057    }
9058    result
9059}
9060
9061/// Spawn one process of `spec` into a slot.
9062///
9063/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
9064/// A swap candidate needs a different cgroup from the process it is replacing,
9065/// which is still alive: in the same cgroup the two would be one kill domain,
9066/// and killing a failed candidate could take the incumbent with it.
9067///
9068/// The stderr capture file is `<module_id>.stderr.log` for every process of
9069/// the module, whichever slot it is in, because that is the one file
9070/// `ck module logs` reads. During a swap's overlap both processes append to it;
9071/// the daemon writes whole lines, so the two interleave by line, which is also
9072/// the merged view an operator wants while a swap runs.
9073#[allow(clippy::too_many_arguments)]
9074fn spawn_child_in_slot(
9075    spec: &ModuleSpec,
9076    connection_file_path: Option<&std::path::Path>,
9077    handle: Option<&SupervisorHandle>,
9078    ring: &Arc<Mutex<StderrRing>>,
9079    capture_logs_dir: Option<&std::path::Path>,
9080    roster: &ChildRoster,
9081    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9082    role: SpawnRole,
9083    alternate_slot: bool,
9084) -> Result<SupervisedChild, SuperviseError> {
9085    if let Some(error) = recovery_block_error(spec, roster) {
9086        return Err(error);
9087    }
9088    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9089        return Err(previous_instance_error(spec, pid, reason));
9090    }
9091    if roster.is_closed() {
9092        return Err(SuperviseError::Spawn {
9093            program: spec.program.clone(),
9094            source: io::Error::other("the daemon is shutting down; not starting a new process"),
9095            cgroup_path: None,
9096        });
9097    }
9098    let carried = containment::current(&roster.containment);
9099    let carrying = carries_containment(spec.carry_containment, carried.is_some());
9100    if role != SpawnRole::Plain && carrying {
9101        return Err(SuperviseError::InvalidSpec {
9102            reason: "a carried container cannot overlap a swap candidate".to_owned(),
9103        });
9104    }
9105    #[cfg(unix)]
9106    let launch_identity = if carrying && carried.is_none() {
9107        Some(
9108            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9109                program: spec.program.clone(),
9110                source,
9111                cgroup_path: None,
9112            })?,
9113        )
9114    } else {
9115        None
9116    };
9117    #[cfg(target_os = "linux")]
9118    let cgroup_placement = match &carried {
9119        Some(container) => container.cgroup().map(|(placement, _)| placement),
9120        None => cgroup_placement,
9121    };
9122    #[cfg(unix)]
9123    let group_target = if let Some(container) = carried
9124        .as_ref()
9125        .filter(|container| container.group().is_some())
9126    {
9127        containment::GroupTarget::Pinned(Arc::clone(container))
9128    } else {
9129        #[cfg(target_os = "linux")]
9130        let cgroup_container = carrying && cgroup_placement.is_some();
9131        #[cfg(not(target_os = "linux"))]
9132        let cgroup_container = false;
9133        if cgroup_container {
9134            containment::GroupTarget::NoGroup
9135        } else {
9136            containment::GroupTarget::OwnLeader
9137        }
9138    };
9139    #[cfg(target_os = "linux")]
9140    let cgroup_name = {
9141        // Slot names alone are not kill domains: a retired incumbent may still
9142        // be draining when a later enable/restart spawns into the same slot.
9143        // Decimal entropy keeps the suffix unambiguous; Placement performs
9144        // the module-id escaping and constructs the filesystem path.
9145        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9146            name.to_owned()
9147        } else if cgroup_placement.is_none() {
9148            swap::cgroup_name(&spec.module_id, alternate_slot)
9149        } else {
9150            let nonce = generate_launch_nonce()?;
9151            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9152            // Leave room for byte escaping and the suffix under NAME_MAX. The
9153            // label is only for humans; the nonce identifies the kill domain.
9154            let mut end = spec.module_id.len().min(64);
9155            while !spec.module_id.is_char_boundary(end) {
9156                end -= 1;
9157            }
9158            format!(
9159                "{}_{suffix}",
9160                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9161            )
9162        }
9163    };
9164    #[cfg(not(target_os = "linux"))]
9165    let _ = alternate_slot;
9166    #[cfg(target_os = "macos")]
9167    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9168    #[cfg(windows)]
9169    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9170        program: spec.program.clone(),
9171        source,
9172        cgroup_path: None,
9173    })?;
9174    #[cfg(windows)]
9175    let mut command = Command::new(image_capture.path());
9176    #[cfg(not(any(target_os = "macos", windows)))]
9177    let mut command = Command::new(&spec.program);
9178    command.args(&spec.args);
9179    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
9180    // that is the whole of the intent, so remove that one key rather than the
9181    // environment.
9182    //
9183    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
9184    // and took the POSIX environment with it. Modules spawned that way had no
9185    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
9186    // logging:
9187    //
9188    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
9189    //     both unset it fell back to the temp dir alone and `ck` could not find
9190    //     a daemon running on the same machine from inside any module's process
9191    //     tree — reporting a path the file has never lived at, which reads as
9192    //     "the daemon did not write its file".
9193    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
9194    //     the RELATIVE `.local/share`, so a module deriving its own store path
9195    //     resolved it against its own CWD. That is the store-fragmentation
9196    //     defect the daemon already refuses in config (`parse_doc` rejects a
9197    //     relative `storage.data_home`) arriving by derivation instead.
9198    //   * anything a module spawns inherited it: git without ~/.gitconfig,
9199    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
9200    //     quietly rather than erroring.
9201    //
9202    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
9203    // offered one candidate under /tmp while the file sat in /run/user/1000.
9204    //
9205    // A configured module is unaffected either way: `module_spec()` puts the
9206    // resolved CK_LOG into `spec.env`, which is applied below and therefore
9207    // wins over anything ambient.
9208    apply_child_env(&mut command, spec);
9209    apply_spawn_role(&mut command, role);
9210    let nonce_handoff =
9211        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9212
9213    #[cfg(target_os = "linux")]
9214    let cgroup_path = cgroup_placement
9215        .map(|placement| {
9216            if let Some(container) = &carried {
9217                let path = container
9218                    .cgroup_path()
9219                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9220                placement.existing_module_path(path)?.ok_or_else(|| {
9221                    io::Error::new(
9222                        io::ErrorKind::NotFound,
9223                        "carried cgroup disappeared before replacement",
9224                    )
9225                })
9226            } else {
9227                if carrying {
9228                    containment::prepare_cgroup_path(placement, &cgroup_name)
9229                } else {
9230                    placement.module_path(&cgroup_name)
9231                }
9232            }
9233        })
9234        .transpose()
9235        .map_err(|source| SuperviseError::Cgroup {
9236            module_id: spec.module_id.clone(),
9237            source,
9238        })?;
9239    #[cfg(not(target_os = "linux"))]
9240    let cgroup_path: Option<PathBuf> = None;
9241    #[cfg(unix)]
9242    let mut admission_gate = if carrying {
9243        Some(
9244            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9245                SuperviseError::Spawn {
9246                    program: spec.program.clone(),
9247                    source,
9248                    cgroup_path: cgroup_path.clone(),
9249                }
9250            })?,
9251        )
9252    } else {
9253        None
9254    };
9255    #[cfg(target_os = "macos")]
9256    exec_ack.install(command.as_std_mut());
9257    #[cfg(target_os = "linux")]
9258    if let Some(path) = &cgroup_path {
9259        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9260            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9261                remove_module_cgroup(placement, &cgroup_name);
9262            }
9263            return Err(error);
9264        }
9265    }
9266
9267    let output_sink = if let Some(logs_dir) = capture_logs_dir {
9268        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9269        match ChildOutputSink::open(&path, capture_retention(spec)) {
9270            Ok(sink) => sink,
9271            Err(error) => {
9272                warn!(
9273                    module_id = %spec.module_id,
9274                    path = %path.display(),
9275                    error = %error,
9276                    "could not open child output capture file; forwarding to stderr"
9277                );
9278                ChildOutputSink::Stderr
9279            }
9280        }
9281    } else {
9282        ChildOutputSink::Stderr
9283    };
9284
9285    command.stdout(Stdio::piped());
9286    command.stderr(Stdio::piped());
9287    command.kill_on_drop(!carrying);
9288    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
9289    // before exec). In the daemon's group, a service manager that kills the
9290    // job's process group when the daemon exits (launchd's default) killed
9291    // every module at the same moment its control connection closed, so no
9292    // module ever ran its EOF teardown on a daemon stop. Outside that group a
9293    // module is reached only by the daemon: the EOF it sees when its
9294    // connection closes, and the bounded stop in `child_roster` for anything
9295    // still running after that. On Linux this composes with the cgroup
9296    // placement above: that is a pre_exec write to cgroup.procs, std performs
9297    // setpgid in the child before running pre_exec callbacks, and the two
9298    // change independent process attributes.
9299    //
9300    // stdin is /dev/null because a process outside the terminal's foreground
9301    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
9302    // by hand would otherwise hand down. Under a service manager stdin is
9303    // already /dev/null.
9304    #[cfg(unix)]
9305    command.process_group(
9306        carried
9307            .as_ref()
9308            .map(|container| container.verified_join_group())
9309            .transpose()
9310            .map_err(|source| SuperviseError::Spawn {
9311                program: spec.program.clone(),
9312                source,
9313                cgroup_path: cgroup_path.clone(),
9314            })?
9315            .flatten()
9316            .unwrap_or(0) as i32,
9317    );
9318    command.stdin(Stdio::null());
9319    #[cfg(unix)]
9320    if let Some(gate) = &mut admission_gate {
9321        gate.install_placement(&mut command);
9322    }
9323    // The LAST pre-exec step, after the cgroup placement above: installing the
9324    // nonce at descriptor 3 replaces whatever the child had there, which could
9325    // be the descriptor an earlier step writes through.
9326    #[cfg(unix)]
9327    if let Some(handoff) = nonce_handoff {
9328        handoff.install_last(command.as_std_mut());
9329    }
9330    #[cfg(not(any(unix, windows)))]
9331    let _ = nonce_handoff;
9332
9333    // Containment, step 1 of 3 (issue #109): create the child suspended so it
9334    // cannot run a single instruction -- and therefore cannot spawn a
9335    // grandchild -- before it is in the job. See `contain_spawned_child` for the
9336    // other two steps and why the window matters.
9337    #[cfg(windows)]
9338    {
9339        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9340            program: spec.program.clone(),
9341            source,
9342            cgroup_path: None,
9343        })?;
9344        subc_jobobject::suspend_on_create_async(&mut command);
9345    }
9346    // Timestamp before the spawn call: on Unix the child can serve before that
9347    // call even returns. This conservative bound never labels an older child
9348    // as a replacement for a request received during process creation.
9349    let spawned_at = Instant::now();
9350    #[cfg(all(test, unix))]
9351    containment_tests::record_spawn_attempt(spec, &mut command);
9352    #[cfg(unix)]
9353    let spawned = match admission_gate {
9354        Some(gate) => gate
9355            .spawn(
9356                command,
9357                spec,
9358                roster,
9359                launch_identity,
9360                #[cfg(target_os = "linux")]
9361                cgroup_placement
9362                    .zip(cgroup_path.as_ref())
9363                    .map(|(placement, path)| {
9364                        (placement.clone(), cgroup_name.clone(), path.clone())
9365                    }),
9366            )
9367            .map(|(child, admitted)| (child, Some(admitted))),
9368        None => command.spawn().map(|child| (child, None)),
9369    };
9370    #[cfg(not(unix))]
9371    let spawned = command.spawn();
9372    let spawned = match spawned {
9373        Ok(child) => child,
9374        Err(source) => {
9375            if carried
9376                .as_ref()
9377                .and_then(|container| container.group())
9378                .is_some()
9379            {
9380                containment::terminate_blocking(&roster.containment);
9381            }
9382            #[cfg(target_os = "linux")]
9383            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9384                remove_module_cgroup(placement, &cgroup_name);
9385            }
9386            return Err(SuperviseError::Spawn {
9387                program: spec.program.clone(),
9388                source,
9389                cgroup_path,
9390            });
9391        }
9392    };
9393    #[cfg(unix)]
9394    let (mut child, mut admitted) = spawned;
9395    #[cfg(unix)]
9396    let group_target = admitted
9397        .as_ref()
9398        .map(|attempt| attempt.group_target())
9399        .unwrap_or(group_target);
9400    #[cfg(not(unix))]
9401    let mut child = spawned;
9402    #[cfg(test)]
9403    spawn_record_tests::after_create(&child, spec);
9404    // The parent must close its writer now: the acknowledgement pipe reports EOF
9405    // only when every writer is gone, and the child's copy closes when the
9406    // trampoline replaces itself with the module. Command holds only an integer
9407    // in its pre_exec callback, not another writer.
9408    #[cfg(target_os = "macos")]
9409    drop(exec_ack);
9410
9411    // Bind while the child is still suspended, before job assignment resumes it.
9412    // Keeping the executable handle open across CreateProcessW denies rename
9413    // and deletion, so the file cannot be exchanged during spawn. Bind that
9414    // file ID to the process handle returned by spawn, never a reopened PID.
9415    #[cfg(windows)]
9416    let windows_image = match image_capture.bind_tokio(&child) {
9417        Ok(image) => Arc::new(image),
9418        Err(source) => {
9419            kill_and_reap_rejected_child(&mut child);
9420            return Err(SuperviseError::Spawn {
9421                program: spec.program.clone(),
9422                source,
9423                cgroup_path: None,
9424            });
9425        }
9426    };
9427    // Assign the suspended child to its module job; resume only after its
9428    // process identity has been durably recorded for orphan recovery.
9429    #[cfg(windows)]
9430    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9431    let spawned_at_ms = unix_ms_now();
9432    #[cfg(not(windows))]
9433    let spawned_from = spec.program.clone();
9434    #[cfg(windows)]
9435    let spawned_from = windows_image.path().to_path_buf();
9436    #[cfg(not(windows))]
9437    let spawned_file_identity = spawned_file_identity(&spawned_from);
9438    #[cfg(windows)]
9439    let spawned_file_identity = Some(SpawnedFileIdentity {
9440        device: 0,
9441        inode: 0,
9442        windows_image: Some(Arc::clone(&windows_image)),
9443    });
9444    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9445        program: spec.program.clone(),
9446        source: io::Error::other("spawned child exposed no live pid"),
9447        cgroup_path: cgroup_path.clone(),
9448    })?;
9449    #[cfg(windows)]
9450    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9451    #[cfg(not(windows))]
9452    let process_start_time = admitted
9453        .as_ref()
9454        .map(|admitted| admitted.birth)
9455        .or_else(|| crate::provenance::process_start_time(pid));
9456    #[cfg(windows)]
9457    let process_start_time = Some(windows_image.start_time());
9458    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9459    #[cfg(all(test, target_os = "macos"))]
9460    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9461    // Record the committed kernel image, not the configured pathname: PATH
9462    // lookup and shebang interpretation can select a different executable.
9463    // Linux's posix_spawn can return before that image is installed, so the
9464    // observer also waits for the kernel's first-exec flag to clear.
9465    #[cfg(not(windows))]
9466    let recorded_image = observe_spawned_image(pid);
9467    // spawn() confirms only the first exec, into the trampoline. Never persist
9468    // the trampoline image; the asynchronous acknowledgement publishes the
9469    // module image once the trampoline has replaced itself with the module.
9470    #[cfg(target_os = "macos")]
9471    let recorded_image = if privacy_exec.is_some() {
9472        None
9473    } else {
9474        recorded_image
9475    };
9476    #[cfg(target_os = "linux")]
9477    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9478    #[cfg(not(target_os = "linux"))]
9479    let recorded_cgroup_name = None;
9480    #[cfg(unix)]
9481    let preadmitted = admitted
9482        .as_mut()
9483        .and_then(|admitted| admitted.leader.take());
9484    #[cfg(not(unix))]
9485    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9486    let roster_guard = if let Some(guard) = preadmitted {
9487        Ok(guard)
9488    } else {
9489        roster.admit(
9490            spec.module_id.clone(),
9491            pid,
9492            spec.protocol,
9493            process_start_time,
9494            crate::child_roster::RecordedIdentity {
9495                container: None,
9496                #[cfg(unix)]
9497                carried: None,
9498                #[cfg(not(windows))]
9499                start_time: recorded_image.map(|image| image.start_time),
9500                #[cfg(windows)]
9501                start_time: process_start_time,
9502                #[cfg(not(windows))]
9503                executable: recorded_image
9504                    .and_then(|image| image.executable)
9505                    .map(crate::live_children::ExecutableIdentity::from),
9506                #[cfg(windows)]
9507                executable: Some(crate::live_children::ExecutableIdentity::from(
9508                    windows_image.identity(),
9509                )),
9510                cgroup_name: recorded_cgroup_name,
9511                #[cfg(target_os = "linux")]
9512                cgroup_placement: cgroup_placement.cloned(),
9513            },
9514        )
9515    };
9516    let roster_guard = match roster_guard {
9517        Ok(guard) => guard,
9518        Err(failure) => {
9519            return failure.with_cleanup(|source| {
9520                #[cfg(windows)]
9521                if let Some(job) = &job {
9522                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9523                }
9524                #[cfg(target_os = "linux")]
9525                kill_module_cgroup(cgroup_placement, &cgroup_name);
9526                #[cfg(unix)]
9527                let rejected_group = group_target.clone();
9528                kill_and_reap_rejected_child(
9529                    &mut child,
9530                    #[cfg(unix)]
9531                    rejected_group,
9532                );
9533                #[cfg(target_os = "linux")]
9534                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9535                    remove_module_cgroup(placement, &cgroup_name);
9536                }
9537                Err(SuperviseError::Spawn {
9538                    program: spec.program.clone(),
9539                    source: io::Error::other(format!(
9540                        "could not durably record module child: {source}"
9541                    )),
9542                    cgroup_path,
9543                })
9544            });
9545        }
9546    };
9547    #[cfg(unix)]
9548    if admitted.is_some() {
9549        if let Some(image) = recorded_image {
9550            if let Err(source) = roster_guard.confirm_executable(image) {
9551                kill_and_reap_rejected_child(&mut child, group_target.clone());
9552                return Err(SuperviseError::Spawn {
9553                    program: spec.program.clone(),
9554                    source,
9555                    cgroup_path,
9556                });
9557            }
9558        }
9559    }
9560    // The pre-spawn check can pass just before shutdown begins. Ownership is
9561    // published at the start of admission, even while a record write is queued.
9562    // If shutdown already saw an empty roster before that publication, this
9563    // check sees the roster closed and ends the child before starting a module.
9564    if roster.is_closed() {
9565        if carrying {
9566            #[cfg(unix)]
9567            let rejected_group = admitted
9568                .as_ref()
9569                .map(|admitted| admitted.rejection_target())
9570                .unwrap_or_else(|| group_target.clone());
9571            #[cfg(windows)]
9572            if let Some(job) = &job {
9573                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9574            }
9575            kill_and_reap_rejected_child(
9576                &mut child,
9577                #[cfg(unix)]
9578                rejected_group,
9579            );
9580            #[cfg(all(test, unix))]
9581            let _ =
9582                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9583            drop(roster_guard);
9584            return Err(SuperviseError::Spawn {
9585                program: spec.program.clone(),
9586                source: io::Error::other(
9587                    "the daemon began shutting down during carried admission; ended the container",
9588                ),
9589                cgroup_path,
9590            });
9591        }
9592        // This child was never admitted, so there is no module protocol shutdown to wait for.
9593        #[cfg(target_os = "linux")]
9594        kill_module_cgroup(cgroup_placement, &cgroup_name);
9595        if let Err(error) = child.start_kill() {
9596            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9597        }
9598        #[cfg(target_os = "linux")]
9599        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9600            // This spawn was never admitted, so shutdown has no roster entry
9601            // to await. Do not detach its cleanup: the runtime could exit
9602            // before that task reaps the rejected child and removes its group.
9603            while matches!(child.try_wait(), Ok(None)) {
9604                std::thread::yield_now();
9605            }
9606            if matches!(
9607                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9608                subc_cgroup::KillOutcome::Killed
9609            ) {
9610                if let Ok(path) = placement.module_path(&cgroup_name) {
9611                    while std::fs::read_to_string(path.join("cgroup.events"))
9612                        .ok()
9613                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9614                    {
9615                        std::thread::yield_now();
9616                    }
9617                }
9618            }
9619            remove_module_cgroup(placement, &cgroup_name);
9620        }
9621        drop(roster_guard);
9622        return Err(SuperviseError::Spawn {
9623            program: spec.program.clone(),
9624            source: io::Error::other(
9625                "the daemon began shutting down while this process was starting; ended it",
9626            ),
9627            cgroup_path,
9628        });
9629    }
9630
9631    let stdout_pump = match child.stdout.take() {
9632        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9633        None => {
9634            warn!(
9635                module_id = %spec.module_id,
9636                "spawned child exposed no stdout pipe; file capture will be incomplete"
9637            );
9638            None
9639        }
9640    };
9641    let stderr_pump = match child.stderr.take() {
9642        Some(stderr) => {
9643            let generation = ring
9644                .lock()
9645                .unwrap_or_else(|poisoned| poisoned.into_inner())
9646                .begin_process();
9647            Some(StderrPump {
9648                task: tokio::spawn(pump_stderr_to(
9649                    stderr,
9650                    Arc::clone(ring),
9651                    generation,
9652                    output_sink,
9653                )),
9654                generation,
9655            })
9656        }
9657        None => {
9658            // Spawning succeeded but the pipe did not materialise. Recording it as
9659            // uncaptured keeps the tail honest: the alternative is an empty tail
9660            // that reads as a module which printed nothing.
9661            ring.lock()
9662                .unwrap_or_else(|poisoned| poisoned.into_inner())
9663                .mark_not_captured("stderr pipe was not available on spawn");
9664            warn!(
9665                module_id = %spec.module_id,
9666                "spawned child exposed no stderr pipe; tail will be unavailable"
9667            );
9668            None
9669        }
9670    };
9671
9672    let exec = Arc::new(crate::activity::ExecOutcome::new());
9673    #[cfg(target_os = "macos")]
9674    let pending_exec = privacy_exec.is_some();
9675    #[cfg(not(target_os = "macos"))]
9676    let pending_exec = false;
9677    if !pending_exec {
9678        exec.finish(crate::activity::ExecState::CommittedExec);
9679    }
9680    #[cfg(windows)]
9681    let subtree_target = job
9682        .as_ref()
9683        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9684        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9685    #[cfg(target_os = "linux")]
9686    let subtree_target = cgroup_path
9687        .map(crate::subtree_resources::Target::Cgroup)
9688        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9689    #[cfg(not(any(target_os = "linux", windows)))]
9690    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9691    let mut supervised = SupervisedChild {
9692        subtree_target,
9693        child: containment::ProcessChild::plain(child, pid),
9694        exec: crate::activity::ExecOwner(exec),
9695        #[cfg(windows)]
9696        nonce_delivery,
9697        protocol: spec.protocol,
9698        #[cfg(target_os = "linux")]
9699        module_id: cgroup_name,
9700        #[cfg(target_os = "linux")]
9701        cgroup_placement: cgroup_placement.cloned(),
9702        #[cfg(windows)]
9703        job,
9704        stdout_pump,
9705        stderr_pump,
9706        stderr_ring: Arc::clone(ring),
9707        spawned_at,
9708        registration_deadline: None,
9709        registration_watch: None,
9710        spawned_at_ms,
9711        spawned_from,
9712        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9713        spawned_file_identity,
9714        process_start_time,
9715        process_identity,
9716        pid,
9717        roster_guard: Some(roster_guard),
9718        #[cfg(target_os = "macos")]
9719        privacy_exec,
9720        #[cfg(target_os = "macos")]
9721        report_ready: Arc::new(OnceLock::new()),
9722        spawn_failure: None,
9723    };
9724    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9725        supervised.child.reject(
9726            #[cfg(unix)]
9727            group_target,
9728        );
9729        #[cfg(all(test, unix))]
9730        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9731        return Err(error);
9732    }
9733    #[cfg(unix)]
9734    if let Some(admitted) = admitted.take() {
9735        admitted.disarm();
9736    }
9737    #[cfg(windows)]
9738    if let Err(error) = resume_suspended_child(pid, spec) {
9739        containment::terminate_blocking(&roster.containment);
9740        let _ = supervised.child.start_kill();
9741        return Err(error);
9742    }
9743    Ok(supervised)
9744}
9745
9746#[cfg(test)]
9747mod spawn_record_tests {
9748    use super::*;
9749
9750    thread_local! {
9751        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9752    }
9753
9754    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9755        LAST_PID.set(child.id());
9756        #[cfg(windows)]
9757        if let Some((_, address)) = _spec
9758            .env
9759            .iter()
9760            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9761        {
9762            use std::io::{Read, Write};
9763            let mut stream = std::net::TcpStream::connect(address).unwrap();
9764            stream
9765                .write_all(&child.id().unwrap().to_ne_bytes())
9766                .unwrap();
9767            let mut release = [0];
9768            stream.read_exact(&mut release).unwrap();
9769        }
9770    }
9771
9772    #[cfg(windows)]
9773    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9774        if spec
9775            .env
9776            .iter()
9777            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9778        {
9779            Err(io::Error::other("injected module job assignment failure"))
9780        } else {
9781            Ok(())
9782        }
9783    }
9784
9785    fn spec() -> ModuleSpec {
9786        ModuleSpec {
9787            module_id: "spawn-record-worker".into(),
9788            #[cfg(unix)]
9789            program: "/bin/sleep".into(),
9790            #[cfg(windows)]
9791            program: std::env::current_exe().unwrap(),
9792            #[cfg(unix)]
9793            args: vec!["60".into()],
9794            #[cfg(windows)]
9795            args: vec![
9796                "--exact".into(),
9797                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9798                "--nocapture".into(),
9799            ],
9800            env: vec![],
9801            reserved: false,
9802            reserved_prefixes: vec![],
9803            protocol: ModuleProtocol::None,
9804            overlap: Default::default(),
9805            carry_containment: false,
9806        }
9807    }
9808
9809    #[tokio::test]
9810    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9811        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9812        let supervisor =
9813            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9814                .with_live_children_record(home.join("live-children.json"));
9815        crate::live_children::fail_next_file_sync();
9816        let result = supervisor.spawn(spec());
9817        let failed = result.is_err();
9818        if let Ok(module) = result {
9819            module.stop().await.unwrap();
9820        }
9821        assert!(failed, "record failure must fail spawn");
9822        let pid = LAST_PID.get().expect("the real OS child was created");
9823        assert!(
9824            subc_os::Process::open(pid).unwrap().is_none(),
9825            "child {pid} survived rejected admission or was not reaped"
9826        );
9827    }
9828
9829    #[cfg(target_os = "macos")]
9830    #[test]
9831    #[ignore = "manual on-device latency measurement"]
9832    fn durable_record_write_latency_50() {
9833        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9834        let path = home.join("live-children.json");
9835        let children = vec![crate::live_children::LiveChild {
9836            module_id: "latency-worker".into(),
9837            pid: std::process::id(),
9838            protocol: ModuleProtocol::None,
9839            start_time: Some(1),
9840            executable: None,
9841            cgroup_name: None,
9842            container: None,
9843        }];
9844        let mut samples = Vec::new();
9845        for _ in 0..50 {
9846            let start = std::time::Instant::now();
9847            crate::live_children::write_record(&path, &children).unwrap();
9848            samples.push(start.elapsed());
9849        }
9850        samples.sort();
9851        eprintln!(
9852            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9853            (samples[24] + samples[25]) / 2,
9854            samples[47]
9855        );
9856    }
9857
9858    #[cfg(windows)]
9859    #[test]
9860    fn windows_store_touch_fixture() {
9861        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9862            std::fs::write(path, b"module ran").unwrap();
9863        }
9864    }
9865
9866    #[cfg(windows)]
9867    #[tokio::test]
9868    async fn windows_daemon_fixture() {
9869        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9870            return;
9871        };
9872        let mut spec = spec();
9873        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9874        let supervisor =
9875            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9876        let _module = supervisor.spawn(spec).unwrap();
9877    }
9878
9879    #[cfg(windows)]
9880    #[test]
9881    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9882        use std::io::Read;
9883        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9884        // An outer job models a scheduler/service host. The daemon's root and
9885        // root job must nest without requiring breakaway from this job.
9886        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9887        command
9888            .args([
9889                "--exact",
9890                "supervise::spawn_record_tests::windows_daemon_fixture",
9891                "--nocapture",
9892            ])
9893            .env(
9894                "SUBC_TEST_DAEMON_BARRIER",
9895                listener.local_addr().unwrap().to_string(),
9896            );
9897        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9898        listener.set_nonblocking(true).unwrap();
9899        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9900        let mut stream = loop {
9901            if let Ok((stream, _)) = listener.accept() {
9902                break stream;
9903            }
9904            assert!(
9905                daemon.child.try_wait().unwrap().is_none(),
9906                "daemon failed before rendezvous"
9907            );
9908            assert!(
9909                std::time::Instant::now() < deadline,
9910                "daemon did not reach CreateProcess failpoint"
9911            );
9912            std::thread::yield_now();
9913        };
9914        stream
9915            .set_read_timeout(Some(Duration::from_secs(30)))
9916            .unwrap();
9917        let mut pid = [0; 4];
9918        stream.read_exact(&mut pid).unwrap();
9919        let pid = u32::from_ne_bytes(pid);
9920        assert!(subc_jobobject::process_exists(pid));
9921        // Terminate only the daemon, not its outer job: otherwise the host job
9922        // would also kill a child whose daemon never joined its root job.
9923        daemon.child.kill().unwrap();
9924        daemon.child.wait().unwrap();
9925        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9926        if !exited {
9927            daemon.job.terminate().unwrap();
9928        }
9929        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9930    }
9931
9932    #[cfg(windows)]
9933    #[tokio::test]
9934    async fn windows_containment_failure_never_resumes() {
9935        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9936        let sentinel = home.join("store-touched");
9937        let mut spec = spec();
9938        spec.env
9939            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9940        spec.env.push((
9941            "SUBC_TEST_STORE_TOUCH".into(),
9942            sentinel.display().to_string(),
9943        ));
9944        let supervisor =
9945            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9946        let result = supervisor.spawn(spec);
9947        // A version that resumes despite assignment failure must have time to
9948        // execute its sentinel before dropping the returned module kills it.
9949        if let Ok(module) = result {
9950            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9951                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9952                    tokio::task::yield_now().await;
9953                }
9954            })
9955            .await;
9956            let _ = module.stop().await;
9957            panic!(
9958                "assignment failure resumed a module; store touched={}",
9959                sentinel.exists()
9960            );
9961        }
9962        assert!(
9963            !sentinel.exists(),
9964            "refused module executed store-touch sentinel"
9965        );
9966        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9967    }
9968
9969    #[cfg(windows)]
9970    #[tokio::test]
9971    async fn windows_nested_daemon_fixture() {
9972        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9973            return;
9974        };
9975        let mut spec = spec();
9976        spec.env.push((
9977            "SUBC_TEST_STORE_TOUCH".into(),
9978            sentinel.to_str().unwrap().into(),
9979        ));
9980        let supervisor =
9981            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9982        let module = supervisor
9983            .spawn(spec)
9984            .expect("per-module job must nest below daemon and host jobs");
9985        tokio::time::timeout(Duration::from_secs(10), async {
9986            while !PathBuf::from(&sentinel).exists() {
9987                tokio::task::yield_now().await;
9988            }
9989        })
9990        .await
9991        .unwrap();
9992        module.stop().await.unwrap();
9993    }
9994
9995    #[cfg(windows)]
9996    #[test]
9997    fn windows_module_job_nests_under_host_job() {
9998        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9999        let sentinel = home.join("nested-store-touched");
10000        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
10001        command
10002            .args([
10003                "--exact",
10004                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
10005                "--nocapture",
10006            ])
10007            .env("SUBC_TEST_NESTED_STORE", &sentinel);
10008        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
10009        assert!(daemon.child.wait().unwrap().success());
10010        assert!(sentinel.exists(), "nested protocol:none module never ran");
10011    }
10012}
10013
10014#[cfg(target_os = "linux")]
10015pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10016    use subc_cgroup::KillOutcome;
10017    match subc_cgroup::kill_module(placement, module_id) {
10018        KillOutcome::Killed => {}
10019        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10020            debug!(
10021                module_id,
10022                "cgroup tree kill unavailable; using direct-child kill"
10023            );
10024        }
10025        KillOutcome::IoError { path, error } => {
10026            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10027        }
10028    }
10029}
10030
10031/// Assign a freshly spawned Windows child without ever resuming on failure.
10032///
10033/// Steps 2 and 3 of the suspended-create contract: the job is created and the
10034/// child assigned **while it is still suspended** (step 1 is
10035/// `suspend_on_create_async` at the spawn site). The caller resumes only after
10036/// the child's roster admission is durable. Failed containment kills and reaps
10037/// the suspended child instead of running an unprotected module.
10038#[cfg(windows)]
10039fn contain_spawned_child(
10040    child: &mut Child,
10041    spec: &ModuleSpec,
10042    carried: Option<&Arc<containment::CarriedContainment>>,
10043) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10044    let assignment = || {
10045        let job = match carried {
10046            Some(container) => container.job(),
10047            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10048        };
10049        #[cfg(test)]
10050        spawn_record_tests::assignment_failpoint(spec)?;
10051        job.lock()
10052            .unwrap_or_else(|p| p.into_inner())
10053            .assign(child)?;
10054        Ok::<_, io::Error>(job)
10055    };
10056    match assignment() {
10057        Ok(job) => Ok(Some(job)),
10058        Err(source) => {
10059            kill_and_reap_rejected_child(child);
10060            Err(SuperviseError::Spawn {
10061                program: spec.program.clone(),
10062                source: io::Error::other(format!(
10063                    "Windows module job containment failed: {source}"
10064                )),
10065                cgroup_path: None,
10066            })
10067        }
10068    }
10069}
10070
10071/// Rejection is synchronous: do not return a spawn error while the child can
10072/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
10073fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10074    #[cfg(unix)]
10075    group.signal_rejected(child);
10076    let _ = child.start_kill();
10077    while matches!(child.try_wait(), Ok(None)) {
10078        std::thread::yield_now();
10079    }
10080}
10081
10082/// Resume a suspended child, killing it if it cannot be started.
10083///
10084/// A suspended process holds a pid and does nothing, so there is no useful
10085/// state to return: the caller gets an error and the spawn fails.
10086#[cfg(windows)]
10087fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10088    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10089        // Kill it here rather than leaving a suspended process for the caller
10090        // to notice; `kill_on_drop` would eventually do this, but the module
10091        // would have been reported as running in between.
10092        let _ = std::process::Command::new("taskkill.exe")
10093            .args(["/PID", &pid.to_string(), "/T", "/F"])
10094            .stdin(Stdio::null())
10095            .stdout(Stdio::null())
10096            .stderr(Stdio::null())
10097            .status();
10098        return Err(SuperviseError::Spawn {
10099            program: spec.program.clone(),
10100            source,
10101            cgroup_path: None,
10102        });
10103    }
10104    Ok(())
10105}
10106
10107#[cfg(target_os = "linux")]
10108fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10109    match placement.remove_module(module_id) {
10110        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10111        Err(error) => warn!(
10112            module_id,
10113            error = %error,
10114            "could not remove module cgroup after process exit; continuing teardown"
10115        ),
10116    }
10117}
10118
10119#[cfg(target_os = "linux")]
10120const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10121#[cfg(target_os = "linux")]
10122const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10123
10124#[cfg(target_os = "linux")]
10125async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10126    let path = placement.module_path(module_id).ok();
10127    if path.as_ref().is_some_and(|path| {
10128        std::fs::read_to_string(path.join("cgroup.events"))
10129            .ok()
10130            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10131    }) {
10132        remove_module_cgroup(placement, module_id);
10133        return;
10134    }
10135
10136    // Freeze before enumerating: a frozen task cannot exit and release its PID
10137    // for reuse by a process outside this module while SIGTERM is being sent.
10138    if terminate_reaped_cgroup(placement, module_id).await {
10139        if let Some(path) = &path {
10140            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10141            while cgroup_is_populated(path) && Instant::now() < deadline {
10142                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10143            }
10144        }
10145    }
10146
10147    // Reaping the direct child is not proof its descendants exited. Kill any
10148    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
10149    // cgroup empty before rmdir.
10150    if matches!(
10151        subc_cgroup::kill_module(Some(placement), module_id),
10152        subc_cgroup::KillOutcome::Killed
10153    ) {
10154        if let Some(path) = &path {
10155            while cgroup_is_populated(path) {
10156                sleep(Duration::from_millis(1)).await;
10157            }
10158        }
10159    }
10160    remove_module_cgroup(placement, module_id);
10161}
10162
10163#[cfg(target_os = "linux")]
10164fn cgroup_is_populated(path: &std::path::Path) -> bool {
10165    std::fs::read_to_string(path.join("cgroup.events"))
10166        .ok()
10167        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10168}
10169
10170#[cfg(target_os = "linux")]
10171async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10172    let frozen =
10173        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10174            .await
10175        {
10176            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10177            outcome => {
10178                tracing::debug!(
10179                    module_id,
10180                    ?outcome,
10181                    "cgroup freeze failed; skipping SIGTERM grace"
10182                );
10183                return false;
10184            }
10185        };
10186    let listed = match frozen.processes() {
10187        subc_cgroup::ProcsOutcome::Listed(pids) => {
10188            for pid in pids.into_iter().filter(|pid| *pid > 1) {
10189                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10190                    if let Err(error) =
10191                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10192                    {
10193                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10194                    }
10195                }
10196            }
10197            true
10198        }
10199        outcome => {
10200            tracing::debug!(
10201                module_id,
10202                ?outcome,
10203                "cgroup process listing failed; skipping SIGTERM grace"
10204            );
10205            false
10206        }
10207    };
10208    match frozen.thaw() {
10209        subc_cgroup::ThawOutcome::Thawed => listed,
10210        outcome => {
10211            tracing::debug!(
10212                module_id,
10213                ?outcome,
10214                "cgroup thaw failed; proceeding to cgroup.kill"
10215            );
10216            false
10217        }
10218    }
10219}
10220
10221#[cfg(target_os = "linux")]
10222fn apply_cgroup_placement(
10223    command: &mut Command,
10224    spec: &ModuleSpec,
10225    path: &std::path::Path,
10226) -> Result<(), SuperviseError> {
10227    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10228        module_id: spec.module_id.clone(),
10229        source,
10230    })
10231}
10232
10233fn capture_retention(spec: &ModuleSpec) -> Retention {
10234    let defaults = Retention::default();
10235    let value = |name: &str| {
10236        spec.env
10237            .iter()
10238            .rev()
10239            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10240    };
10241    Retention {
10242        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10243            .and_then(|value| value.parse().ok())
10244            .unwrap_or(defaults.max_file_mb),
10245        keep: value(CAPTURE_KEEP_ENV)
10246            .and_then(|value| value.parse().ok())
10247            .unwrap_or(defaults.keep),
10248        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10249            .and_then(|value| value.parse().ok())
10250            .unwrap_or(defaults.max_age_days),
10251    }
10252}
10253
10254/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
10255/// module's registration to the exact process the supervisor spawned.
10256fn generate_launch_nonce() -> Result<String, SuperviseError> {
10257    let mut bytes = [0u8; 32];
10258    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10259        reason: source.to_string(),
10260    })?;
10261    let mut hex = String::with_capacity(64);
10262    for b in bytes {
10263        use std::fmt::Write;
10264        let _ = write!(hex, "{b:02x}");
10265    }
10266    Ok(hex)
10267}
10268
10269/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
10270/// signal about how many leading bytes matched.
10271fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10272    if a.len() != b.len() {
10273        return false;
10274    }
10275    let mut diff = 0u8;
10276    for (x, y) in a.iter().zip(b.iter()) {
10277        diff |= x ^ y;
10278    }
10279    diff == 0
10280}
10281
10282/// The kernel's image after an acknowledged exec, shared by ordinary launches
10283/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
10284/// not identities inferred from a configured pathname.
10285#[cfg(not(windows))]
10286fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10287    let process = subc_os::Process::open(pid).ok().flatten()?;
10288    #[cfg(target_os = "linux")]
10289    return sample_after_linux_exec(
10290        || {
10291            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10292            let committed = linux_exec_committed(&stat);
10293            #[cfg(test)]
10294            spawn_image_tests::report_exec_state(committed);
10295            committed
10296        },
10297        || process.observe(),
10298    );
10299    #[cfg(not(target_os = "linux"))]
10300    process.observe()
10301}
10302
10303/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
10304/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
10305/// installation. Until it clears, /proc/pid/exe can still name the daemon.
10306#[cfg(target_os = "linux")]
10307fn linux_exec_committed(stat: &str) -> Option<bool> {
10308    const PF_FORKNOEXEC: u64 = 0x40;
10309    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10310    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10311}
10312
10313#[cfg(target_os = "linux")]
10314fn sample_after_linux_exec(
10315    mut exec_committed: impl FnMut() -> Option<bool>,
10316    observe: impl FnOnce() -> Option<subc_os::Observation>,
10317) -> Option<subc_os::Observation> {
10318    let deadline = std::time::Instant::now() + Duration::from_secs(1);
10319    while !exec_committed()? {
10320        // A failed or indefinitely stalled exec must not publish the inherited
10321        // image as authority to signal a future orphan. Missing identity keeps
10322        // the existing conservative orphan-sweep behavior.
10323        if std::time::Instant::now() >= deadline {
10324            return None;
10325        }
10326        std::thread::sleep(Duration::from_micros(50));
10327    }
10328    observe()
10329}
10330
10331#[cfg(all(test, target_os = "linux"))]
10332mod spawn_image_tests {
10333    use super::*;
10334    use std::cell::{Cell, RefCell};
10335
10336    enum ExecEvent {
10337        Uncommitted,
10338        Image(Option<subc_os::Observation>),
10339    }
10340
10341    thread_local! {
10342        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10343    }
10344
10345    pub(super) fn report_exec_state(committed: Option<bool>) {
10346        if committed == Some(false) {
10347            EXEC_EVENTS.with(|events| {
10348                if let Some(events) = events.borrow_mut().take() {
10349                    let _ = events.send(ExecEvent::Uncommitted);
10350                }
10351            });
10352        }
10353    }
10354
10355    #[test]
10356    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10357        use std::io::{Read, Write};
10358        use std::os::fd::AsRawFd;
10359
10360        let descriptor = std::fs::File::open("/dev/null").unwrap();
10361        let mut command = std::process::Command::new("/bin/sleep");
10362        command.arg("30");
10363        let (mut ready, mut release) =
10364            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10365                .unwrap();
10366        let spawn = std::thread::spawn(move || command.spawn());
10367        let mut message = [0u8; 8];
10368        ready.read_exact(&mut message).unwrap();
10369        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10370        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10371        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10372        let (events, received) = std::sync::mpsc::channel();
10373        let observer = std::thread::spawn(move || {
10374            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10375            let image = observe_spawned_image(pid);
10376            let _ = events.send(ExecEvent::Image(image));
10377        });
10378        // Release only once the production observer has seen the uncommitted
10379        // exec, or has incorrectly returned the inherited image. Either way,
10380        // release and reap before asserting so a regression cannot leak a child.
10381        let first = received.recv_timeout(Duration::from_secs(5));
10382        release.write_all(&[1]).unwrap();
10383        let mut child = spawn.join().unwrap().unwrap();
10384        let after = match first {
10385            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10386                Ok(ExecEvent::Image(image)) => image,
10387                _ => None,
10388            },
10389            Ok(ExecEvent::Image(image)) => image,
10390            Err(_) => None,
10391        };
10392        observer.join().unwrap();
10393        child.kill().unwrap();
10394        child.wait().unwrap();
10395
10396        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10397        assert_eq!(
10398            inherited.unwrap().executable,
10399            subc_os::file_identity(&std::env::current_exe().unwrap())
10400        );
10401        assert_eq!(
10402            after.unwrap().executable,
10403            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10404        );
10405    }
10406
10407    #[test]
10408    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10409        let committed = Cell::new(false);
10410        let inherited = subc_os::Observation {
10411            start_time: 23,
10412            executable: Some(subc_os::FileIdentity {
10413                device: 29,
10414                inode: 31,
10415            }),
10416        };
10417        let module = subc_os::Observation {
10418            start_time: 23,
10419            executable: Some(subc_os::FileIdentity {
10420                device: 29,
10421                inode: 37,
10422            }),
10423        };
10424        let polls = Cell::new(0);
10425        let image = sample_after_linux_exec(
10426            || {
10427                polls.set(polls.get() + 1);
10428                // Hold the inherited image through the first poll, as in the
10429                // posix_spawn parent-wake window, then commit the module image.
10430                committed.set(polls.get() == 2);
10431                Some(committed.get())
10432            },
10433            || Some(if committed.get() { module } else { inherited }),
10434        );
10435        assert_eq!(image, Some(module));
10436        assert_eq!(polls.get(), 2);
10437    }
10438
10439    #[test]
10440    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10441        assert_eq!(
10442            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10443            Some(false)
10444        );
10445        assert_eq!(
10446            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10447            Some(true)
10448        );
10449        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10450        assert_eq!(linux_exec_committed("17 (a) R"), None);
10451    }
10452
10453    #[test]
10454    fn unreadable_exec_state_never_samples_an_inherited_image() {
10455        assert_eq!(
10456            sample_after_linux_exec(
10457                || None,
10458                || panic!("must not sample before exec confirmation")
10459            ),
10460            None
10461        );
10462    }
10463
10464    #[test]
10465    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10466        assert_eq!(
10467            sample_after_linux_exec(
10468                || Some(false),
10469                || panic!("must not sample after an exec timeout")
10470            ),
10471            None
10472        );
10473    }
10474}
10475
10476/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
10477/// different executable after the no-delete capture was taken.
10478#[cfg(windows)]
10479fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10480    let mut program = spec.program.clone();
10481    if program.is_absolute() || program.components().count() > 1 {
10482        // An explicit path can name a PE image with no extension. Try that
10483        // exact file first, as Command does, before falling back to .exe.
10484        match subc_os::ExecutableCapture::open(&program) {
10485            Ok(capture) => return Ok(capture),
10486            Err(error)
10487                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10488            Err(error) => return Err(error),
10489        }
10490        program.set_extension("exe");
10491        return subc_os::ExecutableCapture::open(&program);
10492    }
10493    if program.extension().is_none() {
10494        program.set_extension("exe");
10495    }
10496    let custom_path = spec
10497        .env
10498        .iter()
10499        .rev()
10500        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10501    let mut directories = Vec::new();
10502    // These are Windows' default executable search locations. A configured
10503    // PATH still overrides the inherited PATH, not the system directories.
10504    if let Some(parent) = std::env::current_exe()?.parent() {
10505        directories.push(parent.to_path_buf());
10506    }
10507    directories.push(std::env::current_dir()?);
10508    if let Some(windows) = std::env::var_os("SystemRoot") {
10509        let windows = PathBuf::from(windows);
10510        directories.push(windows.join("System32"));
10511        directories.push(windows.clone());
10512    }
10513    let path = custom_path
10514        .map(|(_, value)| std::ffi::OsString::from(value))
10515        .or_else(|| std::env::var_os("PATH"));
10516    if let Some(path) = path {
10517        directories.extend(std::env::split_paths(&path));
10518    }
10519    for directory in directories {
10520        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10521            Ok(capture) => return Ok(capture),
10522            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10523            Err(error) => return Err(error),
10524        }
10525    }
10526    Err(io::Error::new(
10527        io::ErrorKind::NotFound,
10528        "resolved executable not found",
10529    ))
10530}
10531
10532fn spawn_and_mark_running(
10533    spec: &ModuleSpec,
10534    runtime: &SupervisorRuntimeConfig,
10535    snapshot: &SharedSnapshot,
10536) -> Result<SupervisedChild, SuperviseError> {
10537    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10538    let registration_watch = watch_registration(&runtime.registry, spec);
10539    #[cfg(unix)]
10540    let joining_group = containment::current(&runtime.child_roster.containment)
10541        .is_some_and(|container| container.group().is_some());
10542    let spawn = || {
10543        spawn_child(
10544            spec,
10545            runtime.connection_file_path.as_deref(),
10546            runtime.supervisor_handle.as_ref(),
10547            &runtime.stderr_ring,
10548            runtime.capture_logs_dir.as_deref(),
10549            &runtime.child_roster,
10550            #[cfg(target_os = "linux")]
10551            runtime.cgroup_placement.as_ref(),
10552        )
10553    };
10554    let result = spawn();
10555    #[cfg(unix)]
10556    let result = if result.is_err()
10557        && joining_group
10558        && !runtime.child_roster.is_closed()
10559        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10560        && runtime
10561            .child_roster
10562            .previous_instance(&spec.module_id)
10563            .is_none()
10564    {
10565        // The failed join already destroyed and verified the old container.
10566        // Try one fresh group as part of this same restart, without spending
10567        // another crash-budget entry. Cgroups and Windows jobs do not retry.
10568        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10569        spawn()
10570    } else {
10571        result
10572    };
10573    let child = result?;
10574    let mut child = child;
10575    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10576    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10577        containment::terminate_blocking(&runtime.child_roster.containment);
10578        return Err(error);
10579    }
10580    if let Some(spawn) = activity_spawn {
10581        spawn.complete(snapshot);
10582    }
10583    Ok(child)
10584}
10585
10586struct RegistrationWatch {
10587    changes: watch::Receiver<u64>,
10588    baseline: u64,
10589}
10590
10591// Capture before spawning: a fast child's accepted HELLO can disappear from the
10592// active registry before the monitor first runs. A pre-existing registration
10593// must not satisfy a new process's deadline either.
10594fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10595    if spec.protocol != ModuleProtocol::Subc {
10596        return None;
10597    }
10598    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10599    let baseline = *changes.borrow();
10600    Some(RegistrationWatch { changes, baseline })
10601}
10602
10603fn arm_registration_deadline(
10604    child: &mut SupervisedChild,
10605    budget: Duration,
10606    registration_watch: Option<RegistrationWatch>,
10607) {
10608    child.registration_deadline =
10609        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10610    child.registration_watch = registration_watch;
10611}
10612
10613async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10614    if let Some(container) = child.child.container() {
10615        container.stop_members().await;
10616        child.kill_leader()?;
10617        child.wait().await?;
10618        return Ok(());
10619    }
10620    #[cfg(unix)]
10621    {
10622        // Children lead their own process groups. Signal the whole group so a
10623        // helper stuck before HELLO cannot survive its supervised parent.
10624        let group = unreaped_registration_group(child)?;
10625        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10626        // Process-group safety rule: only signal a group while its leader is
10627        // our unreaped child. Even a zombie leader reserves its pid/group id;
10628        // reaping it before KILL would let that id name an unrelated group.
10629        // Sleep without waiting so TERM-ignoring helpers get the full grace.
10630        sleep(crate::child_roster::TERM_TO_KILL).await;
10631        let group = unreaped_registration_group(child)?;
10632        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10633    }
10634    // Windows terminates the containing Job Object as well as the direct
10635    // child. Linux also kills the child's cgroup when placement is enabled.
10636    child.start_kill()?;
10637    child.wait().await?;
10638    Ok(())
10639}
10640
10641#[cfg(unix)]
10642fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10643    if let Some(container) = child.child.container() {
10644        return container
10645            .group()
10646            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10647            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10648    }
10649    // The group id comes only from the pid recorded at spawn, when the child
10650    // created its own group. Never admit init's group or a negative group id.
10651    let group = i32::try_from(child.pid)
10652        .ok()
10653        .filter(|pid| *pid > 1)
10654        .and_then(rustix::process::Pid::from_raw)
10655        .ok_or_else(|| {
10656            io::Error::new(
10657                io::ErrorKind::InvalidInput,
10658                "invalid supervised process group",
10659            )
10660        })?;
10661    if child.child.id() != Some(child.pid) {
10662        return Err(io::Error::new(
10663            io::ErrorKind::InvalidInput,
10664            "refusing process-group signal after supervised child was reaped",
10665        ));
10666    }
10667    Ok(group)
10668}
10669
10670enum RegistrationWaitOutcome {
10671    Registered,
10672    Exited(ExitReport),
10673    TimedOut,
10674}
10675
10676struct ReloadRegistrationFailure {
10677    exit_report: ExitReport,
10678    reason: String,
10679}
10680
10681#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10682enum BusyGaugeObservation {
10683    Quiescent,
10684    Busy,
10685    Omitted,
10686}
10687
10688/// What one health answer said about one declared busy gauge.
10689#[derive(Debug, Clone, PartialEq, Eq)]
10690enum GaugeVerdict {
10691    /// The gauge read 0: this gauge is not what holds the drain.
10692    Zero,
10693    /// The gauge read a positive count.
10694    NonZero(u64),
10695    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10696    /// a negative number). It counts as busy, because the daemon cannot tell
10697    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10698    NotACount(String),
10699    /// The health answer had no metrics object, or one without this gauge.
10700    Missing,
10701    /// The probe produced no health answer to read; carries the probe error's
10702    /// evidence label (`no-answer` for a probe that timed out).
10703    ProbeFailed(&'static str),
10704}
10705
10706impl GaugeVerdict {
10707    fn reads_quiet(&self) -> bool {
10708        matches!(self, Self::Zero)
10709    }
10710}
10711
10712impl fmt::Display for GaugeVerdict {
10713    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10714        match self {
10715            Self::Zero => f.write_str("zero"),
10716            Self::NonZero(value) => write!(f, "non_zero({value})"),
10717            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10718            Self::Missing => f.write_str("missing_from_health_body"),
10719            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10720        }
10721    }
10722}
10723
10724/// The longest JSON text kept for a gauge value that is not a count, so a
10725/// module reporting a large object as a gauge cannot flood the drain log.
10726const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10727
10728/// Read each declared gauge out of one health answer's metrics.
10729fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10730    let metrics = metrics.and_then(Value::as_object);
10731    gauges
10732        .iter()
10733        .map(
10734            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10735                None => GaugeVerdict::Missing,
10736                Some(value) => match value.as_u64() {
10737                    Some(0) => GaugeVerdict::Zero,
10738                    Some(count) => GaugeVerdict::NonZero(count),
10739                    None => {
10740                        let mut text = value.to_string();
10741                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10742                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10743                            while !text.is_char_boundary(end) {
10744                                end -= 1;
10745                            }
10746                            text.truncate(end);
10747                            text.push_str("...");
10748                        }
10749                        GaugeVerdict::NotACount(text)
10750                    }
10751                },
10752            },
10753        )
10754        .collect()
10755}
10756
10757/// The drain's overall reading of one health answer: omitted when any
10758/// declared gauge is missing, quiescent only when every gauge reads 0.
10759fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10760    if verdicts
10761        .iter()
10762        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10763    {
10764        BusyGaugeObservation::Omitted
10765    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10766        BusyGaugeObservation::Quiescent
10767    } else {
10768        BusyGaugeObservation::Busy
10769    }
10770}
10771
10772/// One declared gauge's history over a drain: the latest verdict on it, and
10773/// the latest value actually read for it (which a later failed probe or
10774/// missing gauge does not erase). Times are offsets from the start of the
10775/// drain's wait.
10776#[derive(Debug, Clone, Default, PartialEq, Eq)]
10777struct GaugeTrace {
10778    latest: Option<(GaugeVerdict, Duration)>,
10779    last_value: Option<(GaugeVerdict, Duration)>,
10780}
10781
10782/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10783/// its ceiling can say in its timeout line which declared gauge held it and
10784/// why. Without it, a gauge that read busy, a gauge the module never
10785/// reported, and a probe that never got an answer all looked the same: a
10786/// drain that ended at its ceiling with nothing in flight.
10787#[derive(Debug, Clone, Default, PartialEq, Eq)]
10788struct DrainGaugeTrace {
10789    /// Health probes the drain sent.
10790    probes: u32,
10791    /// Each declared gauge, in declaration order.
10792    gauges: Vec<(String, GaugeTrace)>,
10793}
10794
10795impl DrainGaugeTrace {
10796    fn new(gauges: &[String]) -> Self {
10797        Self {
10798            probes: 0,
10799            gauges: gauges
10800                .iter()
10801                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10802                .collect(),
10803        }
10804    }
10805
10806    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10807    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10808        self.probes = self.probes.saturating_add(1);
10809        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10810            trace.latest = Some((verdict.clone(), at));
10811            if matches!(
10812                verdict,
10813                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10814            ) {
10815                trace.last_value = Some((verdict.clone(), at));
10816            }
10817        }
10818    }
10819}
10820
10821/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10822/// declared gauge, separated by ` | `. A value is shown as its verdict
10823/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10824/// and whether that counted as busy.
10825impl fmt::Display for DrainGaugeTrace {
10826    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10827        if self.gauges.is_empty() {
10828            return f.write_str("none declared");
10829        }
10830        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10831            if index > 0 {
10832                f.write_str(" | ")?;
10833            }
10834            write!(f, "{name}: ")?;
10835            match &trace.latest {
10836                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10837                None => f.write_str("never probed")?,
10838            }
10839            match &trace.last_value {
10840                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10841                None => f.write_str("; no value ever read")?,
10842            }
10843        }
10844        Ok(())
10845    }
10846}
10847
10848pub(crate) fn declared_busy_gauges(
10849    registry: &Registry,
10850    module_id: &str,
10851) -> Result<Vec<String>, SuperviseError> {
10852    busy_gauges_of(
10853        registry
10854            .get_module(module_id)
10855            .map_err(SuperviseError::Registry)?,
10856    )
10857}
10858
10859/// [`declared_busy_gauges`] for the registration a connection holds, in any
10860/// slot: after cutover the incumbent is no longer the id's active
10861/// registration, and its own manifest is the one that names its gauges.
10862fn declared_busy_gauges_for_connection(
10863    registry: &Registry,
10864    connection_id: ConnectionId,
10865) -> Result<Vec<String>, SuperviseError> {
10866    busy_gauges_of(
10867        registry
10868            .get_module_by_connection(connection_id)
10869            .map_err(SuperviseError::Registry)?,
10870    )
10871}
10872
10873fn busy_gauges_of(
10874    registration: Option<crate::registry::ModuleRegistration>,
10875) -> Result<Vec<String>, SuperviseError> {
10876    let Some(registration) = registration else {
10877        return Ok(Vec::new());
10878    };
10879    let Some(self_signals) = registration.manifest.self_signals else {
10880        return Ok(Vec::new());
10881    };
10882
10883    let mut gauges = Vec::new();
10884    for declaration in self_signals {
10885        if declaration.kind != SelfSignalKind::Busy {
10886            continue;
10887        }
10888        match declaration.anchored_to {
10889            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10890                gauges.extend(declared)
10891            }
10892            _ => {
10893                // An invalid Busy anchor is fail-safe: the empty name cannot be
10894                // present in a conforming health report, so this drain stays busy.
10895                gauges.push(String::new());
10896            }
10897        }
10898    }
10899    Ok(gauges)
10900}
10901
10902/// Wait for `endpoint` to have nothing in flight and, when the module declares
10903/// busy gauges, for a health probe to report them quiet. The probe is addressed
10904/// by `scope`: a swap's superseded incumbent must be asked about its own
10905/// gauges, and by module id the probe would reach the promoted candidate.
10906/// While the gauges read busy, the probe repeats every
10907/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10908/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10909/// timeout line.
10910#[allow(clippy::too_many_arguments)]
10911async fn wait_for_forwarding_quiescence(
10912    forwarding: &ForwardingTable,
10913    module_id: &str,
10914    runtime: &SupervisorRuntimeConfig,
10915    endpoint: crate::ModuleEndpointId,
10916    deadline: Instant,
10917    busy_gauges: &[String],
10918    scope: DrainScope,
10919    trace: &mut DrainGaugeTrace,
10920) -> Result<bool, SuperviseError> {
10921    let wait_started = Instant::now();
10922    let mut gauges_quiescent = busy_gauges.is_empty();
10923    let mut next_probe_at = wait_started;
10924    let mut omission_reported = false;
10925
10926    loop {
10927        let now = Instant::now();
10928        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10929            let report = match scope {
10930                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10931                DrainScope::Endpoint(endpoint) => {
10932                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10933                }
10934            };
10935            let verdicts = match &report {
10936                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10937                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10938            };
10939            trace.record(&verdicts, wait_started.elapsed());
10940            gauges_quiescent = match report {
10941                Ok(_) => match busy_gauge_observation(&verdicts) {
10942                    BusyGaugeObservation::Quiescent => true,
10943                    BusyGaugeObservation::Busy => false,
10944                    BusyGaugeObservation::Omitted => {
10945                        // Once per drain: a module that never reports a gauge
10946                        // it declared would otherwise say so on every probe.
10947                        if !omission_reported {
10948                            forwarding
10949                                .counters()
10950                                .increment_drains_with_undeclared_gauge();
10951                            let missing = busy_gauges
10952                                .iter()
10953                                .zip(&verdicts)
10954                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10955                                .map(|(gauge, _)| gauge.as_str())
10956                                .collect::<Vec<_>>()
10957                                .join(",");
10958                            warn!(
10959                                module_id,
10960                                missing_gauges = %missing,
10961                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10962                            );
10963                            omission_reported = true;
10964                        }
10965                        false
10966                    }
10967                },
10968                Err(err) => {
10969                    warn!(
10970                        module_id,
10971                        error = %err,
10972                        "drain health.check did not produce declared busy gauges; treating module as busy"
10973                    );
10974                    false
10975                }
10976            };
10977            // Counted from the probe's completion: the probe above was awaited,
10978            // so no second probe can start while one is still outstanding.
10979            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10980        }
10981
10982        let in_flight = forwarding
10983            .endpoint_in_flight_count(endpoint)
10984            .map_err(SuperviseError::Forwarding)?;
10985        if in_flight == 0 && gauges_quiescent {
10986            return Ok(true);
10987        }
10988
10989        let now = Instant::now();
10990        if now >= deadline {
10991            return Ok(false);
10992        }
10993        let mut wait = deadline
10994            .saturating_duration_since(now)
10995            .min(REGISTRY_RELEASE_POLL);
10996        if !busy_gauges.is_empty() {
10997            wait = wait.min(next_probe_at.saturating_duration_since(now));
10998        }
10999        sleep(wait).await;
11000    }
11001}
11002
11003/// The wait between drain busy-gauge probes: the larger of
11004/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
11005/// drain's own in-flight poll interval.
11006fn drain_busy_reprobe_interval() -> Duration {
11007    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
11008}
11009
11010/// Logs why a drain gave up: the drain waited its full timeout without the
11011/// module going quiet, and teardown is being forced.
11012///
11013/// The line names two kinds of evidence, because a bare "timed out" hides
11014/// causes that need different fixes:
11015/// - `holdouts`: the requests and routes still open when the timeout hit,
11016///   and on which connections. One stuck call is a module bug; a held stream
11017///   on every session is a module that should end its streams when it
11018///   receives `module.draining`.
11019/// - `gauges`: for each busy gauge the module declares, its latest reading
11020///   and how many probes ran. This separates a module that kept reporting
11021///   itself busy, one whose health answer never contained the gauge, and one
11022///   that never answered the probes, which otherwise all look like a timeout
11023///   with nothing in flight.
11024fn warn_drain_timed_out(
11025    module_id: &str,
11026    drain_timeout: Duration,
11027    reason: RouteCloseReason,
11028    holdouts: &crate::forwarding::DrainHoldouts,
11029    gauges: &DrainGaugeTrace,
11030) {
11031    warn!(
11032        module_id = %module_id,
11033        waited = ?drain_timeout,
11034        ?reason,
11035        held_requests = holdouts.requests,
11036        held_routes = holdouts.routes,
11037        total_routes = holdouts.total_routes,
11038        top_connections = ?holdouts.top_connections,
11039        // `module_channel:corr`, so the module can find each held request
11040        // in its own log; capped, so `held_requests` is the full count.
11041        held = %holdouts
11042            .held
11043            .iter()
11044            .map(|(channel, corr)| format!("{channel}:{corr}"))
11045            .collect::<Vec<_>>()
11046            .join(","),
11047        gauge_probes = gauges.probes,
11048        busy_gauges = %gauges,
11049        "route drain timed out before request quiescence; forcing teardown"
11050    );
11051}
11052
11053/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
11054///
11055/// `Ok` is always honest and passed straight through -- the wait actually measured
11056/// in-flight state. `Err` means the wait produced no measurement at all (the
11057/// forwarding table's lock was poisoned), so `false` is reported as the one honest
11058/// constant: the drain did not complete. Never recomputed from route state, never a
11059/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
11060fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11061    match wait_result {
11062        Ok(drained) => *drained,
11063        Err(_) => false,
11064    }
11065}
11066
11067fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11068    for released in released_routes {
11069        let frame = match Frame::build_with_version(
11070            released.negotiated_ver,
11071            FrameType::Goodbye,
11072            control_flags(),
11073            released.channel,
11074            released.epoch,
11075            0,
11076            Vec::new(),
11077        ) {
11078            Ok(frame) => frame,
11079            Err(err) => {
11080                warn!(
11081                    route_channel = released.channel,
11082                    error = %err,
11083                    "failed to build supervisor drain route GOODBYE frame"
11084                );
11085                continue;
11086            }
11087        };
11088        if !released.close_on_delivery_failure() {
11089            crate::forwarding::send_module_route_goodbye(
11090                &forwarding.counters(),
11091                &released.sink,
11092                frame,
11093                released.module_id.as_deref(),
11094                "supervisor drain",
11095            );
11096            continue;
11097        }
11098        if let Err(err) = released.sink.try_send(frame) {
11099            warn!(
11100                target_connection_id = released.connection_id.get(),
11101                route_channel = released.channel,
11102                error = %err,
11103                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11104            );
11105            let _ = forwarding.escalate_client_delivery_failure(
11106                released.connection_id,
11107                released.channel,
11108                released.epoch,
11109                CloseReason::new(
11110                    "route_goodbye_delivery_failed",
11111                    format!(
11112                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11113                        released.channel
11114                    ),
11115                ),
11116                crate::forwarding::UndeliveredFrame {
11117                    module_id: released.module_id.as_deref(),
11118                    sink: &released.sink,
11119                },
11120            );
11121        }
11122    }
11123}
11124
11125async fn send_module_draining(
11126    module_id: &str,
11127    reason: RouteCloseReason,
11128    deadline_ms: u64,
11129    target: &ModuleDrainTarget,
11130    _wait_for_flush: bool,
11131) -> bool {
11132    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11133        reason,
11134        deadline_ms,
11135    }) {
11136        Ok(body) => body,
11137        Err(err) => {
11138            warn!(
11139                module_id,
11140                error = %err,
11141                "failed to encode module draining command"
11142            );
11143            return false;
11144        }
11145    };
11146    let frame = match Frame::build_with_version(
11147        target.negotiated_ver,
11148        FrameType::Push,
11149        control_flags(),
11150        0,
11151        0,
11152        0,
11153        body,
11154    ) {
11155        Ok(frame) => frame,
11156        Err(err) => {
11157            warn!(
11158                module_id,
11159                error = %err,
11160                "failed to build module draining command frame"
11161            );
11162            return false;
11163        }
11164    };
11165    #[cfg(unix)]
11166    if _wait_for_flush {
11167        return target.sink.send_flushed(frame).await.is_ok();
11168    }
11169    if let Err(err) = target.sink.try_send(frame) {
11170        warn!(
11171            module_id,
11172            target_connection_id = target.endpoint.connection_id.get(),
11173            error = %err,
11174            "module draining command was not delivered to peer"
11175        );
11176        return false;
11177    }
11178    true
11179}
11180
11181/// The channel-0 GOODBYE that tells a module its stop is planned.
11182fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11183    match Frame::build_with_version(
11184        negotiated_ver,
11185        FrameType::Goodbye,
11186        control_flags(),
11187        0,
11188        0,
11189        0,
11190        Vec::new(),
11191    ) {
11192        Ok(frame) => Some(frame),
11193        Err(err) => {
11194            warn!(
11195                module_id,
11196                error = %err,
11197                "failed to build module GOODBYE frame"
11198            );
11199            None
11200        }
11201    }
11202}
11203
11204/// Send every registered module connection its module GOODBYE at daemon
11205/// shutdown, then request that connection's close.
11206///
11207/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
11208/// before EOF, so the GOODBYE must reach the socket before the close. A close
11209/// request does not wait for the connection's queued frames: its writer gets a
11210/// bounded grace after the close, is aborted if it overruns it, and the daemon
11211/// process may exit before that grace ends. So with `wait_for_flush`, each
11212/// connection is closed only after its writer has acknowledged writing the
11213/// GOODBYE, or once a short shared budget runs out, so one module that is not
11214/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
11215/// are only queued, for a shutdown the operator has told to stop waiting.
11216/// A connection that is already gone is skipped.
11217#[cfg(unix)]
11218async fn send_module_goodbyes_for_daemon_shutdown(
11219    forwarding: &Arc<ForwardingTable>,
11220    reason: &CloseReason,
11221    wait_for_flush: bool,
11222) {
11223    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11224    let targets = match forwarding.module_connections() {
11225        Ok(targets) => targets,
11226        Err(err) => {
11227            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11228            return;
11229        }
11230    };
11231    let deadline = Instant::now() + GOODBYE_BUDGET;
11232    let mut sends = tokio::task::JoinSet::new();
11233    for target in targets {
11234        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11235            continue;
11236        };
11237        if !wait_for_flush {
11238            if let Err(err) = target.sink.try_send(frame) {
11239                debug!(
11240                    module_id = %target.module_id,
11241                    error = %err,
11242                    "shutdown module GOODBYE was not queued"
11243                );
11244            }
11245            continue;
11246        }
11247        let forwarding = Arc::clone(forwarding);
11248        let reason = reason.clone();
11249        sends.spawn(async move {
11250            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11251                Ok(Ok(())) => {}
11252                Ok(Err(err)) => debug!(
11253                    module_id = %target.module_id,
11254                    error = %err,
11255                    "module connection closed before its shutdown GOODBYE was written"
11256                ),
11257                Err(_) => warn!(
11258                    module_id = %target.module_id,
11259                    budget = ?GOODBYE_BUDGET,
11260                    "shutdown module GOODBYE was not written within its budget; closing anyway"
11261                ),
11262            }
11263            forwarding.request_connection_close(target.endpoint.connection_id, reason);
11264        });
11265    }
11266    // Every task ends by the shared deadline, so this wait is bounded too.
11267    while sends.join_next().await.is_some() {}
11268}
11269
11270fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11271    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11272        return;
11273    };
11274    if let Err(err) = target.sink.try_send(frame) {
11275        warn!(
11276            module_id,
11277            target_connection_id = target.endpoint.connection_id.get(),
11278            error = %err,
11279            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11280        );
11281        forwarding.request_connection_close(
11282            target.endpoint.connection_id,
11283            CloseReason::new(
11284                "module_goodbye_delivery_failed",
11285                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11286            ),
11287        );
11288    }
11289}
11290
11291#[derive(Clone, Copy)]
11292struct ForwardingDrainContext<'a> {
11293    spec: &'a ModuleSpec,
11294    runtime: &'a SupervisorRuntimeConfig,
11295    registry: &'a Registry,
11296    scope: DrainScope,
11297}
11298
11299/// Which process a forwarding drain addresses.
11300#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11301enum DrainScope {
11302    /// Whatever endpoint is active for the module id: every plain stop,
11303    /// restart and reload. Also moves the module's state to `Draining`.
11304    Active,
11305    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
11306    /// module id would resolve to the promoted candidate and leave neither
11307    /// process routable. The module's state is left alone, since the promoted
11308    /// candidate is what it describes and that process is running.
11309    Endpoint(crate::ModuleEndpointId),
11310}
11311
11312/// Whether a child being drained has already been asked to stop by the time
11313/// its drain wait starts.
11314///
11315/// The drain wait is the same budget whatever this says. What it decides is
11316/// whether the supervisor must ask by signal before that wait begins: a child
11317/// that nobody asked will sit out the whole budget and then be SIGKILLed,
11318/// healthy or not.
11319#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11320enum StopNotice {
11321    /// The module was sent `module.draining` and a module GOODBYE over its own
11322    /// registered connection, and stops itself.
11323    SentOverConnection,
11324    /// The forwarding drain found no registered connection for the module: a
11325    /// subc child spawned moments ago that has not sent HELLO yet, or a
11326    /// `protocol: "none"` child, which never registers.
11327    NoConnection,
11328    /// This path sends nothing over the module's connection: the supervisor has
11329    /// no forwarding table, or the caller stops the child without a forwarding
11330    /// drain.
11331    NotSent,
11332}
11333
11334async fn begin_forwarding_drain(
11335    spec: &ModuleSpec,
11336    runtime: &SupervisorRuntimeConfig,
11337    registry: &Registry,
11338    snapshot: &SharedSnapshot,
11339    enabled: Option<bool>,
11340    reason: RouteCloseReason,
11341) -> Result<StopNotice, SuperviseError> {
11342    let Some(forwarding) = runtime.forwarding.as_ref() else {
11343        return Err(SuperviseError::ReloadUnavailable {
11344            module_id: spec.module_id.clone(),
11345            reason: "supervisor was not configured with a forwarding table".to_string(),
11346        });
11347    };
11348
11349    begin_forwarding_drain_with(
11350        forwarding,
11351        ForwardingDrainContext {
11352            spec,
11353            runtime,
11354            registry,
11355            scope: DrainScope::Active,
11356        },
11357        snapshot,
11358        enabled,
11359        reason,
11360        runtime.drain_timeout,
11361    )
11362    .await
11363}
11364
11365async fn begin_forwarding_drain_if_configured(
11366    spec: &ModuleSpec,
11367    runtime: &SupervisorRuntimeConfig,
11368    registry: &Registry,
11369    snapshot: &SharedSnapshot,
11370    enabled: Option<bool>,
11371    reason: RouteCloseReason,
11372) -> Result<StopNotice, SuperviseError> {
11373    begin_forwarding_drain_with_timeout(
11374        spec,
11375        runtime,
11376        registry,
11377        snapshot,
11378        enabled,
11379        reason,
11380        runtime.drain_timeout,
11381    )
11382    .await
11383}
11384
11385/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
11386/// budget, for paths where the operator overrides the module's configured one
11387/// (`supervisor.restart{drain_timeout_ms}`).
11388async fn begin_forwarding_drain_with_timeout(
11389    spec: &ModuleSpec,
11390    runtime: &SupervisorRuntimeConfig,
11391    registry: &Registry,
11392    snapshot: &SharedSnapshot,
11393    enabled: Option<bool>,
11394    reason: RouteCloseReason,
11395    drain_timeout: Duration,
11396) -> Result<StopNotice, SuperviseError> {
11397    let Some(forwarding) = runtime.forwarding.as_ref() else {
11398        return Ok(StopNotice::NotSent);
11399    };
11400
11401    begin_forwarding_drain_with(
11402        forwarding,
11403        ForwardingDrainContext {
11404            spec,
11405            runtime,
11406            registry,
11407            scope: DrainScope::Active,
11408        },
11409        snapshot,
11410        enabled,
11411        reason,
11412        drain_timeout,
11413    )
11414    .await
11415}
11416
11417async fn begin_forwarding_drain_with(
11418    forwarding: &ForwardingTable,
11419    context: ForwardingDrainContext<'_>,
11420    snapshot: &SharedSnapshot,
11421    enabled: Option<bool>,
11422    reason: RouteCloseReason,
11423    drain_timeout: Duration,
11424) -> Result<StopNotice, SuperviseError> {
11425    let ForwardingDrainContext {
11426        spec,
11427        runtime,
11428        registry,
11429        scope,
11430    } = context;
11431    debug_assert_ne!(reason, RouteCloseReason::Crash);
11432    let terminal = matches!(reason, RouteCloseReason::Disable);
11433    let drain_started_at = Instant::now();
11434    let drain_deadline = drain_started_at + drain_timeout;
11435    let deadline_ms =
11436        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11437    let busy_gauges = match scope {
11438        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11439        DrainScope::Endpoint(endpoint) => {
11440            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11441        }
11442    };
11443
11444    // Admission gate first: route.open/commit and route REQUEST admission are closed
11445    // before the first quiescence check, so the outstanding count can only fall.
11446    let gate_started = Instant::now();
11447    let drain_target = match scope {
11448        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11449        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11450    }
11451    .map_err(SuperviseError::Forwarding)?;
11452    // The instant admission closed, and how long taking the forwarding write
11453    // lock to close it took. The timeout line reports only the quiescence
11454    // wait, so without this a drain that started late looked like one that
11455    // started on time.
11456    info!(
11457        module_id = %spec.module_id,
11458        ?reason,
11459        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11460        connected = drain_target.is_some(),
11461        "module drain began; route admission closed"
11462    );
11463    if scope == DrainScope::Active {
11464        update_snapshot(snapshot, Some(&spec.module_id), |state| {
11465            state.state = ModuleState::Draining;
11466            state.draining_to_replace =
11467                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11468            if let Some(enabled) = enabled {
11469                state.enabled = enabled;
11470            }
11471        })?;
11472    }
11473
11474    let Some(target) = drain_target.as_ref() else {
11475        // Nothing was sent: the module has no registered connection to carry
11476        // `module.draining` or a GOODBYE. The caller must not assume the child
11477        // was asked to stop.
11478        return Ok(StopNotice::NoConnection);
11479    };
11480    {
11481        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11482        #[cfg(feature = "test-support")]
11483        forwarding.wait_before_route_closing_for_test().await;
11484        let routes = forwarding
11485            .endpoint_routes(target.endpoint)
11486            .map_err(SuperviseError::Forwarding)?;
11487        let routes_notified = routes.len();
11488        crate::control::send_route_control_pushes(
11489            forwarding,
11490            routes.clone(),
11491            ClientControlPush::RouteClosing {
11492                module_id: spec.module_id.clone(),
11493                channels: Vec::new(),
11494                reason,
11495            },
11496        );
11497        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11498
11499        // `route.closing` was just sent above: from here on every return path,
11500        // including an early one, MUST send `route.closed` before propagating
11501        // anything else. A client holds `closing` as a promise that a verdict is
11502        // coming; leaving early without `closed` strands it waiting forever, since
11503        // `closing` carries no timeout of its own.
11504        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11505        let wait_result = wait_for_forwarding_quiescence(
11506            forwarding,
11507            &spec.module_id,
11508            runtime,
11509            target.endpoint,
11510            drain_deadline,
11511            &busy_gauges,
11512            scope,
11513            &mut gauge_trace,
11514        )
11515        .await;
11516        let drained = drained_after_quiescence_wait(&wait_result);
11517        if let Err(err) = &wait_result {
11518            error!(
11519                module_id = %spec.module_id,
11520                ?reason,
11521                error = %err,
11522                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11523            );
11524        } else if !drained {
11525            // Read before teardown releases the routes.
11526            let holdouts = forwarding
11527                .endpoint_drain_holdouts(target.endpoint)
11528                .unwrap_or_default();
11529            warn_drain_timed_out(
11530                &spec.module_id,
11531                drain_timeout,
11532                reason,
11533                &holdouts,
11534                &gauge_trace,
11535            );
11536        }
11537        crate::control::send_route_control_pushes(
11538            forwarding,
11539            routes,
11540            ClientControlPush::RouteClosed {
11541                module_id: spec.module_id.clone(),
11542                channels: Vec::new(),
11543                reason,
11544                drained,
11545                abandoned: target.abandoned_bindings.len() as u32,
11546                excluded_subscriptions: target.excluded_subscriptions,
11547                terminal: Some(terminal),
11548            },
11549        );
11550        wait_result?;
11551
11552        // `route.closed` has now been sent unconditionally above. From here the
11553        // remaining steps are cleanup (route + module GOODBYE) rather than a
11554        // promise the client is waiting on, but a lock-poisoned
11555        // `release_module_endpoint_routes` would otherwise skip the module
11556        // GOODBYE silently too -- send it before propagating the error.
11557        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11558            Ok(routes) => routes,
11559            Err(err) => {
11560                warn!(
11561                    module_id = %spec.module_id,
11562                    ?reason,
11563                    error = %err,
11564                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11565                );
11566                send_module_goodbye(&spec.module_id, forwarding, target);
11567                return Err(SuperviseError::Forwarding(err));
11568            }
11569        };
11570        let route_goodbye_count = released_routes.len();
11571        send_route_goodbyes(forwarding, released_routes);
11572        send_module_goodbye(&spec.module_id, forwarding, target);
11573
11574        // The drain's happy path was previously silent: every emission above is
11575        // best-effort with only its failure arm logged, so "were consumers told"
11576        // was unprovable from the daemon log (surfaced by a 30-minute consumer
11577        // hang where the open question was exactly whether teardown notice went
11578        // out). One summary line makes that class decidable in one grep.
11579        info!(
11580            module_id = %spec.module_id,
11581            ?reason,
11582            routes_notified,
11583            route_goodbyes = route_goodbye_count,
11584            abandoned_reservations = target.abandoned_bindings.len(),
11585            excluded_subscriptions = target.excluded_subscriptions,
11586            drained,
11587            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11588        );
11589    }
11590
11591    Ok(StopNotice::SentOverConnection)
11592}
11593
11594/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
11595/// the only slot a plain (non-swap) spawn can register into.
11596async fn wait_for_registration_after_reload(
11597    registry: &Registry,
11598    module_id: &str,
11599    snapshot: &SharedSnapshot,
11600    child: &mut SupervisedChild,
11601    wait: Duration,
11602) -> Result<RegistrationWaitOutcome, SuperviseError> {
11603    wait_for_slot_registration(
11604        registry,
11605        crate::registry::RegistrationSlot::Active(module_id),
11606        module_id,
11607        snapshot,
11608        child,
11609        wait,
11610    )
11611    .await
11612}
11613
11614/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
11615///
11616/// Keyed on the slot rather than the bare module id because during a swap the
11617/// id's active slot is already held by the incumbent: an id-keyed wait would
11618/// report the incumbent's registration as the candidate's and a candidate that
11619/// never registers would look registered. A swap candidate waits on
11620/// `crate::registry::RegistrationSlot::Candidate`.
11621async fn wait_for_slot_registration(
11622    registry: &Registry,
11623    slot: crate::registry::RegistrationSlot<'_>,
11624    module_id: &str,
11625    snapshot: &SharedSnapshot,
11626    child: &mut SupervisedChild,
11627    wait: Duration,
11628) -> Result<RegistrationWaitOutcome, SuperviseError> {
11629    let deadline = Instant::now() + wait;
11630    loop {
11631        if registry
11632            .registration(slot)
11633            .map_err(SuperviseError::Registry)?
11634            .is_some()
11635        {
11636            return Ok(RegistrationWaitOutcome::Registered);
11637        }
11638
11639        let now = Instant::now();
11640        if now >= deadline {
11641            return Ok(RegistrationWaitOutcome::TimedOut);
11642        }
11643        let remaining = deadline.saturating_duration_since(now);
11644        let poll = remaining.min(REGISTRY_RELEASE_POLL);
11645
11646        tokio::select! {
11647            wait_result = child.wait() => {
11648                let status = wait_result.map_err(|source| SuperviseError::Wait {
11649                    module_id: module_id.to_string(),
11650                    source,
11651                })?;
11652                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11653                    snapshot,
11654                    child,
11655                    &status,
11656                )));
11657            }
11658            _ = sleep(poll) => {}
11659        }
11660    }
11661}
11662
11663fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11664    // A process stopped for missing its first HELLO did not provide service,
11665    // even if its TERM handler exits zero. Reload also uses this classification
11666    // when a replacement exits before registering.
11667    if exit_report.kind != ExitKind::DeliberateSeverance {
11668        exit_report.kind = ExitKind::Crash;
11669    }
11670    exit_report
11671}
11672
11673async fn handle_reload_child_registration_failure(
11674    spec: &ModuleSpec,
11675    runtime: &SupervisorRuntimeConfig,
11676    registry: &Registry,
11677    process_liveness: &SupervisorProcessLiveness,
11678    snapshot: &SharedSnapshot,
11679    _child: &mut Option<SupervisedChild>,
11680    failure: ReloadRegistrationFailure,
11681) -> Result<(), SuperviseError> {
11682    let ReloadRegistrationFailure {
11683        exit_report,
11684        reason,
11685    } = failure;
11686    match on_child_exit(
11687        spec,
11688        runtime.restart_policy,
11689        registry,
11690        snapshot,
11691        &runtime.terminal_ring,
11692        &runtime.spawn_events,
11693        &runtime.child_roster,
11694        exit_report,
11695        runtime.recovery_is_daemon,
11696    )
11697    .await
11698    {
11699        NextAction::Stop {
11700            registration_released,
11701        } => {
11702            if registration_released {
11703                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11704            }
11705        }
11706        NextAction::Restart { schedule } => {
11707            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11708                schedule.delay
11709            });
11710            if let Some(schedule) = schedule {
11711                log_crash_respawn(&spec.module_id, schedule);
11712            }
11713            schedule_respawn(
11714                runtime,
11715                snapshot,
11716                &spec.module_id,
11717                delay,
11718                RespawnKind::Spawn,
11719                true,
11720            )?;
11721        }
11722    }
11723    Err(SuperviseError::ReloadFailed {
11724        module_id: spec.module_id.clone(),
11725        reason,
11726    })
11727}
11728
11729async fn handle_reload_spawn_failure(
11730    spec: &ModuleSpec,
11731    runtime: &SupervisorRuntimeConfig,
11732    process_liveness: &SupervisorProcessLiveness,
11733    snapshot: &SharedSnapshot,
11734    _child: &mut Option<SupervisedChild>,
11735    reason: String,
11736) -> Result<(), SuperviseError> {
11737    let now = Instant::now();
11738    let mut schedule = None;
11739    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11740        clear_current_process_facts(state);
11741        if state.enabled {
11742            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11743            state.state = if schedule.is_some() {
11744                ModuleState::Restarting
11745            } else {
11746                if runtime
11747                    .failure_context(runtime.recovery_is_daemon)
11748                    .may_retry()
11749                {
11750                    state.failed_retry.schedule(&spec.module_id);
11751                }
11752                ModuleState::Failed
11753            };
11754        } else {
11755            state.state = ModuleState::Disabled;
11756        }
11757    })?;
11758    if let Some(schedule) = schedule {
11759        schedule_respawn(
11760            runtime,
11761            snapshot,
11762            &spec.module_id,
11763            schedule.delay,
11764            RespawnKind::Spawn,
11765            true,
11766        )?;
11767    } else {
11768        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11769    }
11770    Err(SuperviseError::ReloadFailed {
11771        module_id: spec.module_id.clone(),
11772        reason,
11773    })
11774}
11775
11776fn control_flags() -> Flags {
11777    Flags::new(false, Priority::Passive, false)
11778}
11779
11780#[allow(clippy::too_many_arguments)]
11781async fn drain_optional_child(
11782    module_id: &str,
11783    protocol: ModuleProtocol,
11784    stop_notice: StopNotice,
11785    registry: &Registry,
11786    forwarding: Option<&ForwardingTable>,
11787    snapshot: &SharedSnapshot,
11788    terminal_ring: &Arc<Mutex<TerminalRing>>,
11789    spawn_events: &SpawnEventFeed,
11790    child: &mut Option<SupervisedChild>,
11791    drain_timeout: Duration,
11792    final_state: ModuleState,
11793    enabled: Option<bool>,
11794    failure: &FailureContext,
11795) -> Result<(), SuperviseError> {
11796    if let Some(child) = child.take() {
11797        drain_child_to_state(
11798            module_id,
11799            protocol,
11800            stop_notice,
11801            registry,
11802            forwarding,
11803            snapshot,
11804            terminal_ring,
11805            spawn_events,
11806            child,
11807            drain_timeout,
11808            final_state,
11809            enabled,
11810            failure,
11811        )
11812        .await
11813    } else {
11814        update_snapshot(snapshot, Some(module_id), |state| {
11815            state.state = final_state;
11816            if final_state == ModuleState::Failed {
11817                state.failed_retry.failed_at = Some(Instant::now());
11818            }
11819            if let Some(enabled) = enabled {
11820                state.enabled = enabled;
11821            }
11822            clear_current_process_facts(state);
11823        })?;
11824        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11825    }
11826}
11827
11828#[allow(clippy::too_many_arguments)]
11829async fn drain_child_to_state(
11830    module_id: &str,
11831    _protocol: ModuleProtocol,
11832    stop_notice: StopNotice,
11833    registry: &Registry,
11834    forwarding: Option<&ForwardingTable>,
11835    snapshot: &SharedSnapshot,
11836    terminal_ring: &Arc<Mutex<TerminalRing>>,
11837    spawn_events: &SpawnEventFeed,
11838    mut child: SupervisedChild,
11839    drain_timeout: Duration,
11840    final_state: ModuleState,
11841    enabled: Option<bool>,
11842    failure: &FailureContext,
11843) -> Result<(), SuperviseError> {
11844    let protocol = child.protocol;
11845    update_snapshot(snapshot, Some(module_id), |state| {
11846        state.state = ModuleState::Draining;
11847        state.draining_to_replace = final_state == ModuleState::Restarting;
11848        if let Some(enabled) = enabled {
11849            state.enabled = enabled;
11850        }
11851    })?;
11852
11853    // The wait below is the same budget in every case; what differs is
11854    // whether anything has ASKED the child to stop before it starts. Only a
11855    // forwarding drain that reached the module's registered connection has
11856    // (`module.draining`, then a module GOODBYE). Every other child was told
11857    // nothing: a `protocol: "none"` module, which never registers; a subc
11858    // module spawned moments ago that has not sent HELLO yet; or a stop that
11859    // runs no forwarding drain. Without a signal the budget is only a delay
11860    // in front of SIGKILL -- and the not-yet-registered child is the worst
11861    // case, because it registers into a module that is already draining,
11862    // is never told, and is killed while healthy.
11863    if stop_notice != StopNotice::SentOverConnection {
11864        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11865            info!(
11866                module_id,
11867                pid = child.pid,
11868                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11869                "module has no connection yet; requesting stop by signal"
11870            );
11871        }
11872        request_graceful_stop(module_id, &child);
11873    }
11874
11875    let exit_report = match timeout(drain_timeout, child.wait()).await {
11876        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11877        Ok(Err(source)) => {
11878            fail_after_wait_error(
11879                module_id,
11880                snapshot,
11881                terminal_ring,
11882                spawn_events,
11883                format!("failed to wait during drain: {source}"),
11884                failure,
11885            );
11886            return Err(SuperviseError::Wait {
11887                module_id: module_id.to_string(),
11888                source,
11889            });
11890        }
11891        Err(_) => {
11892            // Mirror the sibling arm above: state is already `Draining`, and an
11893            // error propagated from here would strand it there -- a state
11894            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11895            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11896            // `Failed` before `?` keeps the module operator-visible and
11897            // revivable. Trigger is an ESRCH race (process exits between the
11898            // drain timeout firing and the kill) or a post-kill wait failure
11899            // (issue #34).
11900            //
11901            // Logged because the kill is otherwise visible only as signal 9 in
11902            // the terminal ring, and the budget it follows can be long enough
11903            // that consumers see a stretch of refusals with no stated cause.
11904            warn!(
11905                module_id,
11906                pid = child.pid,
11907                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11908                reason = ?final_state,
11909                ?stop_notice,
11910                "drain budget expired before the module exited; killing it"
11911            );
11912            let kill =
11913                if final_state == ModuleState::Restarting && child.child.container().is_some() {
11914                    child.kill_leader()
11915                } else {
11916                    child.start_kill()
11917                };
11918            kill.map_err(|source| {
11919                fail_after_wait_error(
11920                    module_id,
11921                    snapshot,
11922                    terminal_ring,
11923                    spawn_events,
11924                    format!("failed to kill during drain: {source}"),
11925                    failure,
11926                );
11927                SuperviseError::Kill {
11928                    module_id: module_id.to_string(),
11929                    source,
11930                }
11931            })?;
11932            let status = child.wait().await.map_err(|source| {
11933                fail_after_wait_error(
11934                    module_id,
11935                    snapshot,
11936                    terminal_ring,
11937                    spawn_events,
11938                    format!("failed to wait after drain kill: {source}"),
11939                    failure,
11940                );
11941                SuperviseError::Wait {
11942                    module_id: module_id.to_string(),
11943                    source,
11944                }
11945            })?;
11946            classify_reaped_child_exit(snapshot, &child, &status)
11947        }
11948    };
11949
11950    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11951    publish_after_terminal_record(
11952        module_id,
11953        || {
11954            record_terminal_with_detail(
11955                module_id,
11956                terminal_ring,
11957                spawn_events,
11958                &exit_report,
11959                terminal_disposition(final_state),
11960                detail,
11961            )
11962        },
11963        || {
11964            update_snapshot(snapshot, Some(module_id), |state| {
11965                state.state = final_state;
11966                if final_state == ModuleState::Failed {
11967                    state.failed_retry.failed_at = Some(Instant::now());
11968                }
11969                if let Some(enabled) = enabled {
11970                    state.enabled = enabled;
11971                }
11972                clear_current_process_facts(state);
11973                state.last_exit = Some(exit_report.clone());
11974                if exit_report.kind == ExitKind::DeliberateSeverance {
11975                    state.lifetime_restarts += 1;
11976                }
11977            })
11978        },
11979    )?;
11980    child.drain_stderr(module_id).await;
11981
11982    let released =
11983        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11984    child
11985        .finish_exit(
11986            released.is_ok()
11987                && final_state == ModuleState::Restarting
11988                && child.spawn_failure.is_none(),
11989        )
11990        .await
11991        .map_err(|source| SuperviseError::Wait {
11992            module_id: module_id.to_owned(),
11993            source,
11994        })?;
11995    released
11996}
11997
11998/// Ask a child that nothing else has asked to stop, by signal.
11999///
12000/// A registered subc module is asked over its own connection: the drain sends
12001/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
12002/// module GOODBYE, and the module stops itself. A module that speaks no subc
12003/// wire receives none of that, and neither does a subc module that has not
12004/// registered yet, so for them the drain budget would be pure delay in front of
12005/// a SIGKILL -- and for a process with a store to flush (JetStream is the
12006/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
12007/// into a recovery on the next start.
12008///
12009/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
12010/// rule rather than an optimisation: that module's graceful stop is already
12011/// running by the time its child is drained, and a signal would race it.
12012///
12013/// Best-effort by construction. A child that has already exited is the ordinary
12014/// case rather than an error (the kill lands on a reaped or exiting pid), so a
12015/// failure is logged at debug and the wait-then-kill below still decides the
12016/// outcome.
12017#[cfg(unix)]
12018fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12019    let Some(pid) = child
12020        .id()
12021        .and_then(|pid| i32::try_from(pid).ok())
12022        .and_then(rustix::process::Pid::from_raw)
12023    else {
12024        debug!(
12025            module_id,
12026            "no pid to signal for teardown; falling through to the drain wait"
12027        );
12028        return;
12029    };
12030    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12031        Ok(()) => debug!(
12032            module_id,
12033            "sent SIGTERM to a module nothing else asked to stop"
12034        ),
12035        Err(err) => debug!(
12036            module_id,
12037            error = %err,
12038            "SIGTERM to module failed; the drain wait and kill still apply"
12039        ),
12040    }
12041}
12042
12043/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
12044/// Windows does offer need cooperation this supervisor cannot assume: a console
12045/// control event requires sharing a console with the child, and `WM_CLOSE`
12046/// requires the child to pump a message loop. A supervised server process does
12047/// neither, so there is nothing to send and teardown is the wait followed by the
12048/// kill. Emulating a signal here would mean inventing a stop protocol, which is
12049/// the thing `protocol: "none"` exists to avoid.
12050#[cfg(not(unix))]
12051fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12052    debug!(
12053        module_id,
12054        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12055    );
12056}
12057
12058fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12059    match final_state {
12060        ModuleState::Stopped => TerminalDisposition::Stopped,
12061        ModuleState::Disabled => TerminalDisposition::Disabled,
12062        ModuleState::Restarting => TerminalDisposition::Restarting,
12063        ModuleState::Failed => TerminalDisposition::Failed,
12064        ModuleState::Starting
12065        | ModuleState::Running
12066        | ModuleState::Unresponsive
12067        | ModuleState::Draining => {
12068            unreachable!("terminal exits only finish in terminal or restarting states")
12069        }
12070    }
12071}
12072
12073/// Release a reaped child's registration before allowing another spawn.
12074///
12075/// EOF is not a process-lifetime signal: an inherited socket can stay open
12076/// indefinitely, and serial frame dispatch can be waiting on egress instead of
12077/// reading EOF. After the normal release grace, request connection close (which
12078/// cancels both reads and dispatch), then allow one more release grace for the
12079/// connection guard's forwarding cleanup. Never evict a different connection.
12080async fn release_dead_registration(
12081    registry: &Registry,
12082    forwarding: Option<&ForwardingTable>,
12083    snapshot: &SharedSnapshot,
12084    module_id: &str,
12085    failure: &FailureContext,
12086) -> Result<(), SuperviseError> {
12087    let result = async {
12088        let registration = registry
12089            .get_module(module_id)
12090            .map_err(SuperviseError::Registry)?;
12091        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12092            Ok(()) => return Ok(()),
12093            Err(SuperviseError::RegistrationStillActive { .. }) => {}
12094            Err(err) => return Err(err),
12095        }
12096        let pid = lock_snapshot(snapshot)?.reaped_pid;
12097        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12098            warn!(
12099                module_id,
12100                pid,
12101                connection_id = registration.connection_id.get(),
12102                "reaped module registration outlived release grace; closing dead connection"
12103            );
12104            forwarding.request_connection_close(
12105                registration.connection_id,
12106                CloseReason::new(
12107                    "supervised_process_reaped",
12108                    format!("module '{module_id}' pid {pid} exited"),
12109                ),
12110            );
12111            wait_for_slot_registration_release(
12112                registry,
12113                crate::registry::RegistrationSlot::Connection(registration.connection_id),
12114                REGISTRY_RELEASE_TIMEOUT,
12115            )
12116            .await?;
12117        }
12118        wait_for_registration_release(registry, module_id, Duration::ZERO).await
12119    }
12120    .await;
12121    if let Err(err) = &result {
12122        fail_snapshot(snapshot, Some(module_id), None, failure);
12123        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12124    }
12125    result
12126}
12127
12128/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
12129/// plain stop or restart waits for before it spawns a replacement.
12130async fn wait_for_registration_release(
12131    registry: &Registry,
12132    module_id: &str,
12133    wait: Duration,
12134) -> Result<(), SuperviseError> {
12135    wait_for_slot_registration_release(
12136        registry,
12137        crate::registry::RegistrationSlot::Active(module_id),
12138        wait,
12139    )
12140    .await
12141}
12142
12143/// Wait for the registration in `slot` to go away.
12144///
12145/// Keyed on the slot rather than the bare module id because a successful swap
12146/// never empties the id's active slot (the promoted candidate is in it), so an
12147/// id-keyed wait for the incumbent's release would always time out. Draining a
12148/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
12149/// incumbent's connection instead.
12150async fn wait_for_slot_registration_release(
12151    registry: &Registry,
12152    slot: crate::registry::RegistrationSlot<'_>,
12153    wait: Duration,
12154) -> Result<(), SuperviseError> {
12155    let deadline = Instant::now() + wait;
12156    let mut release_events = registration_release_events().subscribe();
12157    let still_active = |registration: &crate::registry::ModuleRegistration| {
12158        SuperviseError::RegistrationStillActive {
12159            module_id: registration.manifest.module_id.clone(),
12160            waited: wait,
12161        }
12162    };
12163    loop {
12164        let _observed_generation = *release_events.borrow_and_update();
12165        let Some(registration) = registry
12166            .registration(slot)
12167            .map_err(SuperviseError::Registry)?
12168        else {
12169            return Ok(());
12170        };
12171
12172        let now = Instant::now();
12173        if now >= deadline {
12174            return Err(still_active(&registration));
12175        }
12176
12177        let remaining = deadline.saturating_duration_since(now);
12178        match timeout(remaining, release_events.changed()).await {
12179            Ok(Ok(())) | Ok(Err(_)) => {}
12180            Err(_) => return Err(still_active(&registration)),
12181        }
12182    }
12183}
12184
12185#[cfg(test)]
12186mod slot_registration_wait_tests {
12187    use super::*;
12188    use crate::registry::{ConnectionId, RegistrationSlot};
12189    use subc_protocol::manifest::ModuleManifest;
12190
12191    #[tokio::test]
12192    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12193        let registry = Arc::new(Registry::default());
12194        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
12195        let runtime = supervisor.runtime_config();
12196        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12197        let spec = ModuleSpec {
12198            module_id: "enable-stale-registration".to_string(),
12199            program: PathBuf::from("/missing/enable-retry-test"),
12200            args: Vec::new(),
12201            env: Vec::new(),
12202            reserved: false,
12203            reserved_prefixes: Vec::new(),
12204            protocol: ModuleProtocol::Subc,
12205            overlap: Default::default(),
12206            carry_containment: false,
12207        };
12208        let connection = ConnectionId::new(90);
12209        registry
12210            .register_with_control_ops(
12211                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12212                1,
12213                connection,
12214                Vec::new(),
12215            )
12216            .unwrap();
12217        let mut child = None;
12218        let err = set_child_enabled(
12219            &spec,
12220            &runtime,
12221            &registry,
12222            &supervisor.process_liveness,
12223            &snapshot,
12224            &mut child,
12225            true,
12226            &runtime.failure_context(false),
12227        )
12228        .await
12229        .unwrap_err();
12230        assert!(matches!(
12231            err,
12232            SuperviseError::RegistrationStillActive { .. }
12233        ));
12234        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12235        assert!(child.is_none());
12236        registry.deregister_connection(connection).unwrap();
12237        let err = set_child_enabled(
12238            &spec,
12239            &runtime,
12240            &registry,
12241            &supervisor.process_liveness,
12242            &snapshot,
12243            &mut child,
12244            true,
12245            &runtime.failure_context(false),
12246        )
12247        .await
12248        .unwrap_err();
12249        assert!(
12250            matches!(err, SuperviseError::Spawn { .. }),
12251            "second enable must attempt a spawn: {err}"
12252        );
12253        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12254    }
12255
12256    const INCUMBENT: u64 = 1;
12257    const CANDIDATE: u64 = 2;
12258
12259    fn swapped_registry() -> Arc<Registry> {
12260        let registry = Arc::new(Registry::default());
12261        let manifest = ModuleManifest::builder("m", "0.1.0").build();
12262        registry
12263            .register_with_control_ops(
12264                manifest.clone(),
12265                1,
12266                ConnectionId::new(INCUMBENT),
12267                Vec::new(),
12268            )
12269            .unwrap();
12270        registry
12271            .register_candidate_with_control_ops(
12272                manifest,
12273                1,
12274                ConnectionId::new(CANDIDATE),
12275                Vec::new(),
12276            )
12277            .unwrap();
12278        registry
12279    }
12280
12281    /// After a promotion the id's active slot is held by the new process, so an
12282    /// id-keyed wait for the incumbent's release can never succeed; the
12283    /// connection-keyed wait completes as soon as the incumbent deregisters.
12284    #[tokio::test]
12285    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12286        let registry = swapped_registry();
12287        registry.promote_candidate("m").unwrap().unwrap();
12288
12289        assert!(matches!(
12290            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
12291            Err(SuperviseError::RegistrationStillActive { .. })
12292        ));
12293
12294        // Still held while the incumbent's connection has not deregistered.
12295        assert!(matches!(
12296            wait_for_slot_registration_release(
12297                &registry,
12298                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12299                Duration::from_millis(50),
12300            )
12301            .await,
12302            Err(SuperviseError::RegistrationStillActive { .. })
12303        ));
12304
12305        let releaser = Arc::clone(&registry);
12306        let release = tokio::spawn(async move {
12307            sleep(Duration::from_millis(20)).await;
12308            releaser
12309                .deregister_connection(ConnectionId::new(INCUMBENT))
12310                .unwrap();
12311            notify_registration_release();
12312        });
12313        wait_for_slot_registration_release(
12314            &registry,
12315            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12316            Duration::from_secs(5),
12317        )
12318        .await
12319        .expect("the incumbent's own registration is released");
12320        release.await.unwrap();
12321        assert!(registry.get_module("m").unwrap().is_some());
12322    }
12323
12324    /// The candidate slot is waited on separately from the active slot: the
12325    /// incumbent's registration neither holds up nor stands in for it.
12326    #[tokio::test]
12327    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12328        let registry = swapped_registry();
12329        assert!(matches!(
12330            wait_for_slot_registration_release(
12331                &registry,
12332                RegistrationSlot::Candidate("m"),
12333                Duration::from_millis(50),
12334            )
12335            .await,
12336            Err(SuperviseError::RegistrationStillActive { .. })
12337        ));
12338        registry
12339            .deregister_connection(ConnectionId::new(CANDIDATE))
12340            .unwrap();
12341        wait_for_slot_registration_release(
12342            &registry,
12343            RegistrationSlot::Candidate("m"),
12344            Duration::from_millis(50),
12345        )
12346        .await
12347        .expect("a candidate slot with no candidate is released");
12348        assert!(registry
12349            .registration(RegistrationSlot::Active("m"))
12350            .unwrap()
12351            .is_some());
12352    }
12353}
12354
12355fn classify_exit(status: &ExitStatus) -> ExitReport {
12356    ExitReport {
12357        kind: if status.success() {
12358            ExitKind::Clean
12359        } else {
12360            ExitKind::Crash
12361        },
12362        code: status.code(),
12363        signal: exit_signal(status),
12364        at_ms: unix_ms_now(),
12365    }
12366}
12367
12368/// The terminal record for a module whose `wait()` call itself errored (e.g. the
12369/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
12370/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
12371/// disposition still must be `Failed` so the terminal ring is not silently missing
12372/// an entry, matching what `fail_snapshot` records for this same arm.
12373fn wait_error_exit_report() -> ExitReport {
12374    ExitReport {
12375        kind: ExitKind::Crash,
12376        code: None,
12377        signal: None,
12378        at_ms: unix_ms_now(),
12379    }
12380}
12381
12382#[cfg(unix)]
12383fn exit_signal(status: &ExitStatus) -> Option<i32> {
12384    use std::os::unix::process::ExitStatusExt;
12385
12386    status.signal()
12387}
12388
12389#[cfg(not(unix))]
12390fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12391    None
12392}
12393
12394/// Give an operator-touched module its full crash budget back.
12395///
12396/// Named for the counter it used to zero; it now empties the in-window ring,
12397/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
12398/// ledger of what happened survives every operator action.
12399fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12400    update_snapshot(snapshot, Some(module_id), |state| {
12401        state.clear_crash_restarts();
12402    })
12403}
12404
12405fn set_running(
12406    snapshot: &SharedSnapshot,
12407    child: &SupervisedChild,
12408    module_id: &str,
12409    spawn_events: &SpawnEventFeed,
12410) -> Result<(), SuperviseError> {
12411    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12412        module_id: Some(module_id.to_string()),
12413    })?;
12414    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12415    if std::mem::take(&mut state.coalesced_restart_pending) {
12416        let generation = state.spawn_generation;
12417        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12418    }
12419    state.drain_disposition_detail = None;
12420    state.spawn_failure = None;
12421    state.waiting_for_previous_pid = None;
12422    state.waiting_for_previous_detail = None;
12423    // Every caller of this is a plain spawn, which always uses the primary key;
12424    // a promoted swap candidate sets the flag itself after this returns.
12425    state.in_alternate_slot = false;
12426    state.configuration_updated_since_spawn = false;
12427    state.spawned_protocol = Some(child.protocol);
12428    state.state = ModuleState::Running;
12429    state.enabled = true;
12430    state.process_alive = true;
12431    state.pid = child.id();
12432    state.subtree_target = child.subtree_target.clone();
12433    // Probe samples belong to one process. Keep module-level escalation and
12434    // late-answer history, but never charge a new child for its predecessor's misses.
12435    state.health.status = SupervisorHealthStatus::Unknown;
12436    state.last_probe_evidence = None;
12437    state.health.consecutive_failures = 0;
12438    state.health.detail = None;
12439    state.health.metrics = None;
12440    state.health.last_probe_ms = None;
12441    state.failed_retry.healthy_since = None;
12442    state.failed_retry.failed_at = None;
12443    state.failed_retry.running_since = Some(child.spawned_at);
12444    #[cfg(windows)]
12445    {
12446        state.pipe_consumed = child
12447            .nonce_delivery
12448            .as_ref()
12449            .map(|delivery| delivery.consumed());
12450    }
12451    #[cfg(target_os = "macos")]
12452    {
12453        state.report_ready = Some(Arc::clone(&child.report_ready));
12454    }
12455    state.spawned_at_ms = Some(child.spawned_at_ms);
12456    state.exec_outcome = Some(Arc::clone(&child.exec.0));
12457    state.spawned_from = Some(child.spawned_from.clone());
12458    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12459    state.spawned_file_identity = child.spawned_file_identity.clone();
12460    state.process_start_time = child.process_start_time;
12461    Ok(())
12462}
12463
12464fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12465    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12466    state.failed_retry.healthy_since = None;
12467    state.failed_retry.running_since = None;
12468    state.waiting_for_previous_pid = None;
12469    state.waiting_for_previous_detail = None;
12470    state.process_alive = false;
12471    state.spawned_protocol = None;
12472    state.pid = None;
12473    #[cfg(windows)]
12474    {
12475        state.pipe_consumed = None;
12476    }
12477    #[cfg(target_os = "macos")]
12478    {
12479        state.report_ready = None;
12480    }
12481    state.spawned_at_ms = None;
12482    state.exec_outcome = None;
12483    state.spawned_from = None;
12484    state.launch_spec_digest = None;
12485    state.spawned_file_identity = None;
12486    state.process_start_time = None;
12487    state.deliberate_severance = None;
12488}
12489
12490#[cfg(test)]
12491fn record_deliberate_severance(
12492    snapshot: &SharedSnapshot,
12493    identity: ProcessIdentity,
12494) -> Result<(), SuperviseError> {
12495    update_snapshot(snapshot, None, |state| {
12496        state.deliberate_severance = Some(identity);
12497    })
12498}
12499
12500fn apply_deliberate_severance_marker(
12501    snapshot: &SharedSnapshot,
12502    exited_identity: Option<ProcessIdentity>,
12503    mut exit_report: ExitReport,
12504) -> ExitReport {
12505    let marker = lock_snapshot(snapshot)
12506        .ok()
12507        .and_then(|mut state| state.deliberate_severance.take());
12508    if marker.is_some() && marker == exited_identity {
12509        exit_report.kind = ExitKind::DeliberateSeverance;
12510    }
12511    exit_report
12512}
12513
12514fn classify_reaped_child_exit(
12515    snapshot: &SharedSnapshot,
12516    child: &SupervisedChild,
12517    status: &ExitStatus,
12518) -> ExitReport {
12519    let _ = update_snapshot(snapshot, None, |state| {
12520        state.reaped_pid = Some(child.pid);
12521        state.spawn_failure = child.spawn_failure.clone();
12522    });
12523    if child.spawn_failure.as_deref() == Some("registration_timeout") {
12524        registration_failure_exit_report(classify_exit(status))
12525    } else {
12526        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12527    }
12528}
12529
12530fn fail_snapshot(
12531    snapshot: &SharedSnapshot,
12532    module_id: Option<&str>,
12533    last_exit: Option<ExitReport>,
12534    failure: &FailureContext,
12535) {
12536    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12537        state.state = ModuleState::Failed;
12538        clear_current_process_facts(state);
12539        if failure.may_retry() && state.enabled {
12540            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12541        } else if failure.shutdown.is_set() {
12542            state.failed_retry.cancel();
12543        }
12544        if let Some(last_exit) = last_exit {
12545            state.last_exit = Some(last_exit);
12546        }
12547    }) {
12548        error!(error = %err, "failed to mark supervisor state failed");
12549    }
12550}
12551
12552fn update_snapshot(
12553    snapshot: &SharedSnapshot,
12554    module_id: Option<&str>,
12555    update: impl FnOnce(&mut SupervisorSnapshot),
12556) -> Result<(), SuperviseError> {
12557    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12558        module_id: module_id.map(ToOwned::to_owned),
12559    })?;
12560    update(&mut state);
12561    Ok(())
12562}
12563
12564const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12565
12566fn lock_snapshot_for_control<'a>(
12567    snapshot: &'a SharedSnapshot,
12568    module_id: &str,
12569    caller: &'static str,
12570) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12571    let started_at = Instant::now();
12572    let guard = lock_snapshot(snapshot)?;
12573    let waited = started_at.elapsed();
12574    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12575        warn!(
12576            module_id = %module_id,
12577            waited_ms = waited.as_millis() as u64,
12578            caller = %caller,
12579            "slow snapshot lock"
12580        );
12581    }
12582    Ok(guard)
12583}
12584
12585fn lock_snapshot(
12586    snapshot: &SharedSnapshot,
12587) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12588    snapshot
12589        .lock()
12590        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12591}
12592
12593#[cfg(test)]
12594mod terminal_history_tests {
12595    include!("supervise_route_open_recovery_tests.rs");
12596    use std::{
12597        path::PathBuf,
12598        sync::Arc,
12599        time::{Duration, Instant},
12600    };
12601
12602    use tokio::{sync::mpsc, time::sleep};
12603
12604    use super::{
12605        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12606        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12607        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12608        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12609        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12610        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12611        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12612        SupervisorHealthStatus, SupervisorSnapshot,
12613    };
12614    // The supervisor's clock, distinct from the `std::time::Instant` these tests
12615    // use for their own wall-clock deadlines: crash-restart instants must be on
12616    // the same clock the production code stamps them with, which is tokio's (and
12617    // is what `start_paused` tests can move).
12618    use super::Instant as ClockInstant;
12619    use crate::{
12620        registry::Registry,
12621        terminal_ring::{TerminalRing, TerminalRingConfig},
12622    };
12623    use std::sync::Mutex;
12624    use subc_control::TerminalDisposition;
12625
12626    /// See the twin in `control.rs` for why this derives the path from
12627    /// `current_exe()` and why the existence check is here: `--lib` alone does
12628    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
12629    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
12630    pub(super) fn fake_aft_stub_path() -> PathBuf {
12631        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
12632        assert!(
12633            path.exists(),
12634            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12635             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12636            path.display()
12637        );
12638        path
12639    }
12640
12641    #[test]
12642    fn reserved_never_spawned_refuses_every_hello() {
12643        // The canary hole: a reserved id whose module has never spawned had NO
12644        // gate entry and admitted anyone -- the reservation protected the nonce
12645        // holder, not the NAME. Now the entry is present with no legitimate
12646        // holder and refuses all comers.
12647        let supervisor = SupervisorHandle::default();
12648        supervisor.apply_identity_configuration(&ModuleSpec {
12649            module_id: "never-spawned".to_string(),
12650            program: PathBuf::from("/usr/bin/false"),
12651            args: Vec::new(),
12652            env: Vec::new(),
12653            reserved: true,
12654            reserved_prefixes: Vec::new(),
12655            protocol: ModuleProtocol::Subc,
12656            overlap: Default::default(),
12657            carry_containment: false,
12658        });
12659        assert!(
12660            supervisor
12661                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12662                .is_err(),
12663            "forged nonce must refuse on a reserved never-spawned id"
12664        );
12665        assert!(
12666            supervisor
12667                .reserved_hello_verification("never-spawned", None)
12668                .is_err(),
12669            "absent nonce must refuse on a reserved never-spawned id"
12670        );
12671        // And a real spawn nonce minted later admits exactly that nonce.
12672        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12673        supervisor.apply_identity_configuration(&ModuleSpec {
12674            module_id: "never-spawned".to_string(),
12675            program: PathBuf::from("/usr/bin/false"),
12676            args: Vec::new(),
12677            env: Vec::new(),
12678            reserved: true,
12679            reserved_prefixes: Vec::new(),
12680            protocol: ModuleProtocol::Subc,
12681            overlap: Default::default(),
12682            carry_containment: false,
12683        });
12684        assert!(supervisor
12685            .reserved_hello_verification("never-spawned", Some("minted"))
12686            .is_ok());
12687        assert!(supervisor
12688            .reserved_hello_verification("never-spawned", Some("forged"))
12689            .is_err());
12690    }
12691
12692    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12693    /// happened, which is what "spent budget" looks like to every reader.
12694    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12695        let now = ClockInstant::now();
12696        for _ in 0..count {
12697            state.crash_restarts.push_back(now);
12698        }
12699    }
12700
12701    /// Age the oldest recorded restart out of `window`, standing in for the hours
12702    /// that would otherwise have to pass. Injecting the instant is the point: a
12703    /// test that slept a real window would take ten minutes and still prove less.
12704    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12705        let aged = state
12706            .crash_restarts
12707            .front()
12708            .expect("a crash restart must be recorded before it can be aged")
12709            .checked_sub(window + Duration::from_secs(1))
12710            .expect("the test clock is far enough from its origin to age an instant");
12711        state.crash_restarts[0] = aged;
12712    }
12713
12714    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12715        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12716        seed_crash_restarts(&mut state, count);
12717        state
12718    }
12719
12720    #[test]
12721    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12722        let policy = RestartPolicy::new(3, Duration::ZERO);
12723        let now = ClockInstant::now();
12724        assert!(daemon_will_restart(
12725            &mut snapshot_with_restarts(true, 2),
12726            &policy,
12727            now
12728        ));
12729        assert!(!daemon_will_restart(
12730            &mut snapshot_with_restarts(true, 3),
12731            &policy,
12732            now
12733        ));
12734        assert!(!daemon_will_restart(
12735            &mut snapshot_with_restarts(false, 0),
12736            &policy,
12737            now
12738        ));
12739    }
12740
12741    #[test]
12742    fn crash_restart_backoff_escalates_with_in_window_count() {
12743        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12744            .with_max_backoff(Duration::from_secs(30));
12745        let now = ClockInstant::now();
12746        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12747        let schedules = (0..4)
12748            .map(|_| {
12749                state
12750                    .next_crash_restart(&policy, now)
12751                    .expect("the test policy allows four crash restarts")
12752            })
12753            .collect::<Vec<_>>();
12754
12755        assert_eq!(
12756            schedules
12757                .iter()
12758                .map(|schedule| schedule.restart_in_window)
12759                .collect::<Vec<_>>(),
12760            vec![0, 1, 2, 3]
12761        );
12762        assert_eq!(
12763            schedules
12764                .iter()
12765                .map(|schedule| schedule.delay)
12766                .collect::<Vec<_>>(),
12767            vec![
12768                Duration::from_millis(100),
12769                Duration::from_secs(1),
12770                Duration::from_secs(10),
12771                Duration::from_secs(30),
12772            ]
12773        );
12774    }
12775
12776    #[test]
12777    fn crash_restart_backoff_resets_after_ring_clear() {
12778        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12779        let now = ClockInstant::now();
12780        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12781        assert_eq!(
12782            state.next_crash_restart(&policy, now).unwrap().delay,
12783            Duration::from_millis(100)
12784        );
12785        assert_eq!(
12786            state.next_crash_restart(&policy, now).unwrap().delay,
12787            Duration::from_secs(1)
12788        );
12789
12790        state.clear_crash_restarts();
12791        let schedule = state
12792            .next_crash_restart(&policy, now)
12793            .expect("a cleared ring must allow another restart");
12794        assert_eq!(schedule.restart_in_window, 0);
12795        assert_eq!(schedule.delay, Duration::from_millis(100));
12796    }
12797
12798    #[test]
12799    fn crash_restart_backoff_ignores_aged_restarts() {
12800        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12801        let now = ClockInstant::now();
12802        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12803        state
12804            .next_crash_restart(&policy, now)
12805            .expect("the first restart is allowed");
12806        state
12807            .next_crash_restart(&policy, now)
12808            .expect("the second restart is allowed");
12809        state.crash_restarts[0] = now
12810            .checked_sub(policy.window + Duration::from_secs(1))
12811            .expect("the fake clock can age a restart past the window");
12812
12813        let schedule = state
12814            .next_crash_restart(&policy, now)
12815            .expect("an aged restart must release its slot");
12816        assert_eq!(schedule.restart_in_window, 1);
12817        assert_eq!(schedule.delay, Duration::from_secs(1));
12818        assert_eq!(state.crash_restarts.len(), 2);
12819    }
12820
12821    /// The budget is a rate: the same three spent restarts refuse a respawn
12822    /// while they are recent and allow one once they have aged past the window.
12823    /// Nothing about the module changed in between, which is the whole point.
12824    #[test]
12825    fn a_budget_spent_before_the_window_no_longer_refuses() {
12826        let policy = RestartPolicy::new(3, Duration::ZERO);
12827        let mut state = snapshot_with_restarts(true, 3);
12828        let now = ClockInstant::now();
12829        assert!(!daemon_will_restart(&mut state, &policy, now));
12830
12831        assert!(daemon_will_restart(
12832            &mut state,
12833            &policy,
12834            now + policy.window + Duration::from_secs(1)
12835        ));
12836        assert!(
12837            state.crash_restarts.is_empty(),
12838            "reading the budget must drop the instants that left the window"
12839        );
12840    }
12841
12842    fn module_with_recovery_snapshot(
12843        state: ModuleState,
12844        enabled: bool,
12845        restart_count: u32,
12846    ) -> SupervisedModule {
12847        let registry = Arc::new(Registry::default());
12848        let supervisor =
12849            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12850        let runtime = supervisor.runtime_config();
12851        let spec = ModuleSpec {
12852            module_id: "recovery-snapshot".to_string(),
12853            program: fake_aft_stub_path(),
12854            args: Vec::new(),
12855            env: Vec::new(),
12856            reserved: false,
12857            reserved_prefixes: Vec::new(),
12858            protocol: ModuleProtocol::Subc,
12859            overlap: Default::default(),
12860            carry_containment: false,
12861        };
12862        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12863        seed_crash_restarts(&mut snapshot, restart_count);
12864        // These tests read synthetic snapshots. A real child and monitor would
12865        // race those reads by replacing the requested state during startup.
12866        let (commands, _rx) = mpsc::channel(4);
12867        SupervisedModule {
12868            inner: Arc::new(SupervisedModuleInner {
12869                module_id: spec.module_id.clone(),
12870                registry,
12871                snapshot: Arc::new(Mutex::new(snapshot)),
12872                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12873                    spec,
12874                    health: runtime.health,
12875                })),
12876                stderr_ring: runtime.stderr_ring,
12877                terminal_ring: runtime.terminal_ring,
12878                commands,
12879                monitor: Mutex::new(None),
12880                restart_policy: runtime.restart_policy,
12881                effective_drain_timeout: runtime.effective_drain_timeout,
12882                provenance_probe: supervisor.provenance_probe.clone(),
12883                retained: runtime.child_roster.retained_view(),
12884            }),
12885        }
12886    }
12887
12888    #[cfg(target_os = "linux")]
12889    #[tokio::test]
12890    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12891        let supervisor =
12892            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12893                .with_cgroup_placement(None);
12894        let result = supervisor.spawn(ModuleSpec {
12895            module_id: "no-cgroup-placement".to_string(),
12896            program: fake_aft_stub_path(),
12897            args: Vec::new(),
12898            env: Vec::new(),
12899            reserved: false,
12900            reserved_prefixes: Vec::new(),
12901            protocol: ModuleProtocol::Subc,
12902            overlap: Default::default(),
12903            carry_containment: false,
12904        });
12905
12906        assert!(
12907            result.is_ok(),
12908            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12909        );
12910    }
12911
12912    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12913    async fn undecided_snapshot_uses_shared_restart_predicate() {
12914        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12915            .will_recover_after_connection_loss()
12916            .unwrap());
12917        assert!(
12918            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12919                .will_recover_after_connection_loss()
12920                .unwrap()
12921        );
12922    }
12923
12924    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12926        assert!(
12927            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12928                .will_recover_after_connection_loss()
12929                .unwrap()
12930        );
12931    }
12932
12933    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12934    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12935        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12936            .will_recover_after_connection_loss()
12937            .unwrap());
12938        assert!(
12939            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12940                .will_recover_after_connection_loss()
12941                .unwrap()
12942        );
12943    }
12944
12945    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12946    async fn warming_snapshot_is_limited_to_startup_phases() {
12947        for state in [
12948            ModuleState::Starting,
12949            ModuleState::Running,
12950            ModuleState::Restarting,
12951        ] {
12952            assert!(
12953                module_with_recovery_snapshot(state, true, 0)
12954                    .is_warming()
12955                    .unwrap(),
12956                "{state:?} should be warming"
12957            );
12958        }
12959        for state in [
12960            ModuleState::Unresponsive,
12961            ModuleState::Draining,
12962            ModuleState::Stopped,
12963            ModuleState::Failed,
12964            ModuleState::Disabled,
12965        ] {
12966            assert!(
12967                !module_with_recovery_snapshot(state, true, 0)
12968                    .is_warming()
12969                    .unwrap(),
12970                "{state:?} should not be warming"
12971            );
12972        }
12973    }
12974
12975    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12976    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12977        let registry = Arc::new(Registry::default());
12978        let supervisor =
12979            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12980        let module = supervisor
12981            .spawn(ModuleSpec {
12982                module_id: "terminal-history".to_string(),
12983                program: fake_aft_stub_path(),
12984                args: Vec::new(),
12985                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12986                reserved: false,
12987                reserved_prefixes: Vec::new(),
12988                protocol: ModuleProtocol::Subc,
12989                overlap: Default::default(),
12990                carry_containment: false,
12991            })
12992            .unwrap();
12993
12994        let deadline = Instant::now() + Duration::from_secs(5);
12995        loop {
12996            let history = module.terminal_history();
12997            if history.entries.len() == 2 {
12998                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12999                assert_eq!(history.dropped, 0);
13000                assert_eq!(
13001                    history
13002                        .entries
13003                        .iter()
13004                        .map(|entry| entry.exit_code)
13005                        .collect::<Vec<_>>(),
13006                    vec![Some(23), Some(23)]
13007                );
13008                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13009                return;
13010            }
13011            assert!(
13012                Instant::now() < deadline,
13013                "module did not retain two terminal exits: {history:?}"
13014            );
13015            sleep(Duration::from_millis(10)).await;
13016        }
13017    }
13018
13019    /// A disable issued while a crash respawn is still backing off must preempt
13020    /// that respawn: the operator's stop wins, the disable must not queue behind
13021    /// the backoff, and the module must never come back up afterwards.
13022    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13023    async fn disable_during_crash_backoff_cancels_pending_respawn() {
13024        let backoff = Duration::from_secs(2);
13025        let supervisor = Supervisor::new_for_test(
13026            Arc::new(Registry::default()),
13027            RestartPolicy::new(10, backoff),
13028        );
13029        let module = supervisor
13030            .spawn(ModuleSpec {
13031                module_id: "disable-during-backoff".to_string(),
13032                program: fake_aft_stub_path(),
13033                args: Vec::new(),
13034                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13035                reserved: false,
13036                reserved_prefixes: Vec::new(),
13037                protocol: ModuleProtocol::Subc,
13038                overlap: Default::default(),
13039                carry_containment: false,
13040            })
13041            .unwrap();
13042
13043        // Wait for the first crash to put the module into its backoff window.
13044        let deadline = Instant::now() + Duration::from_secs(5);
13045        loop {
13046            if module.status().unwrap().state == ModuleState::Restarting {
13047                break;
13048            }
13049            assert!(
13050                Instant::now() < deadline,
13051                "module never entered the crash backoff"
13052            );
13053            sleep(Duration::from_millis(10)).await;
13054        }
13055
13056        let started = Instant::now();
13057        module.set_enabled(false).await.unwrap();
13058        let waited = started.elapsed();
13059
13060        assert!(
13061            waited < backoff / 2,
13062            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13063        );
13064        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13065
13066        // Outlast the backoff: the respawn it was counting down to must never run.
13067        sleep(backoff + Duration::from_millis(500)).await;
13068        let status = module.status().unwrap();
13069        assert_eq!(status.state, ModuleState::Disabled);
13070        assert_eq!(
13071            status.spawn_generation, 1,
13072            "module respawned after the operator disabled it"
13073        );
13074    }
13075
13076    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
13077    /// the shape of nats-server, the program this rule exists for.
13078    #[cfg(unix)]
13079    fn protocol_none_sigterm_exits_clean_spec(
13080        module_id: &str,
13081        dir: &std::path::Path,
13082    ) -> (ModuleSpec, PathBuf, PathBuf) {
13083        let ready = dir.join("ready");
13084        let marker = dir.join("sigterm");
13085        let spec = ModuleSpec {
13086            module_id: module_id.to_string(),
13087            program: fake_aft_stub_path(),
13088            args: Vec::new(),
13089            env: vec![
13090                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13091                (
13092                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13093                    marker.display().to_string(),
13094                ),
13095                (
13096                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13097                    ready.display().to_string(),
13098                ),
13099            ],
13100            reserved: false,
13101            reserved_prefixes: Vec::new(),
13102            protocol: ModuleProtocol::None,
13103            overlap: Default::default(),
13104            carry_containment: false,
13105        };
13106        (spec, ready, marker)
13107    }
13108
13109    /// Wait for a file the child writes, so a signal is never sent before the
13110    /// child's SIGTERM handler is installed (the default disposition would
13111    /// kill it by signal and the exit would not be clean).
13112    #[cfg(unix)]
13113    async fn wait_for_file(path: &std::path::Path) {
13114        let deadline = Instant::now() + Duration::from_secs(10);
13115        while !path.exists() {
13116            assert!(
13117                Instant::now() < deadline,
13118                "{} never appeared",
13119                path.display()
13120            );
13121            sleep(Duration::from_millis(10)).await;
13122        }
13123    }
13124
13125    /// A protocol-none module that exits 0 because something OUTSIDE the
13126    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
13127    /// the crash-path disposition rather than `stopped`.
13128    #[cfg(unix)]
13129    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13130    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13131        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13132        let (spec, ready, marker) =
13133            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13134        let supervisor = Supervisor::new_for_test(
13135            Arc::new(Registry::default()),
13136            RestartPolicy::new(3, Duration::ZERO),
13137        );
13138        let module = supervisor.spawn(spec).unwrap();
13139        wait_for_file(&ready).await;
13140        // The ready file proves the child installed its SIGTERM handler, not
13141        // that the supervisor has processed the privacy trampoline's exec
13142        // acknowledgement. On macOS status withholds the pid until then.
13143        let deadline = Instant::now() + Duration::from_secs(10);
13144        let first_pid = loop {
13145            let status = module.status().unwrap();
13146            if status.state == ModuleState::Running {
13147                if let Some(pid) = status.pid {
13148                    break pid;
13149                }
13150            }
13151            assert!(
13152                Instant::now() < deadline,
13153                "a running module must report its pid after exec confirmation: {status:?}"
13154            );
13155            sleep(Duration::from_millis(10)).await;
13156        };
13157
13158        rustix::process::kill_process(
13159            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13160            rustix::process::Signal::TERM,
13161        )
13162        .unwrap();
13163
13164        let deadline = Instant::now() + Duration::from_secs(10);
13165        let respawned = loop {
13166            let status = module.status().unwrap();
13167            if status.state == ModuleState::Running
13168                && status.pid.is_some_and(|pid| pid != first_pid)
13169            {
13170                break status;
13171            }
13172            assert!(
13173                Instant::now() < deadline,
13174                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13175            );
13176            sleep(Duration::from_millis(10)).await;
13177        };
13178        assert_eq!(respawned.spawn_generation, 2);
13179        assert!(
13180            marker.exists(),
13181            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13182        );
13183
13184        let history = module.terminal_history();
13185        assert_eq!(history.entries.len(), 1, "{history:?}");
13186        let entry = &history.entries[0];
13187        assert_eq!(entry.exit_code, Some(0));
13188        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13189        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13190
13191        module.stop().await.unwrap();
13192    }
13193
13194    /// Repeated unrequested clean exits of a protocol-none module spend the
13195    /// restart budget exactly as crashes do, and the module ends `failed` with
13196    /// the budget named.
13197    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13198    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13199        let supervisor = Supervisor::new_for_test(
13200            Arc::new(Registry::default()),
13201            RestartPolicy::new(1, Duration::ZERO),
13202        );
13203        let module = supervisor
13204            .spawn(ModuleSpec {
13205                module_id: "none-clean-exit-budget".to_string(),
13206                program: fake_aft_stub_path(),
13207                args: Vec::new(),
13208                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13209                reserved: false,
13210                reserved_prefixes: Vec::new(),
13211                protocol: ModuleProtocol::None,
13212                overlap: Default::default(),
13213                carry_containment: false,
13214            })
13215            .unwrap();
13216
13217        // Failed follows the terminal write; this deadline only bounds a hang,
13218        // not an assumed duration for the two launches or their exit recording.
13219        let deadline = Instant::now() + Duration::from_secs(10);
13220        loop {
13221            let status = module.status().unwrap();
13222            if status.state == ModuleState::Failed {
13223                break;
13224            }
13225            assert!(
13226                Instant::now() < deadline,
13227                "module never exhausted its budget: {status:?} {:?}",
13228                module.terminal_history()
13229            );
13230            sleep(Duration::from_millis(10)).await;
13231        }
13232        let history = module.terminal_history();
13233        assert_eq!(
13234            history
13235                .entries
13236                .iter()
13237                .map(|entry| (entry.exit_code, entry.disposition.clone()))
13238                .collect::<Vec<_>>(),
13239            vec![
13240                (Some(0), TerminalDisposition::Restarting),
13241                (Some(0), TerminalDisposition::Failed),
13242            ]
13243        );
13244        let detail = history.entries[1]
13245            .disposition_detail
13246            .as_deref()
13247            .expect("a budget failure names the budget");
13248        assert!(detail.contains("max_restarts=1"), "{detail}");
13249        assert_eq!(module.status().unwrap().spawn_generation, 2);
13250    }
13251
13252    #[test]
13253    fn restart_budget_failure_is_published_after_its_terminal_record() {
13254        use std::io::{Read, Write};
13255        let supervisor = Supervisor::new_for_test(
13256            Arc::new(Registry::default()),
13257            RestartPolicy::new(0, Duration::ZERO),
13258        );
13259        let runtime = supervisor.runtime_config();
13260        let spec = ModuleSpec {
13261            protocol: ModuleProtocol::None,
13262            ..windowed_crash_spec("budget-publication-order")
13263        };
13264        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13265            ModuleState::Running,
13266            true,
13267        )));
13268        lock_snapshot(&snapshot).unwrap().process_alive = true;
13269        let reaped_snapshot = snapshot.clone();
13270        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13271        let history = runtime.terminal_ring.clone();
13272        let reap = std::thread::spawn(move || {
13273            tokio::runtime::Builder::new_current_thread()
13274                .enable_all()
13275                .build()
13276                .unwrap()
13277                .block_on(on_child_exit(
13278                    &spec,
13279                    runtime.restart_policy,
13280                    &supervisor.registry,
13281                    &reaped_snapshot,
13282                    &runtime.terminal_ring,
13283                    &runtime.spawn_events,
13284                    &runtime.child_roster,
13285                    ExitReport {
13286                        kind: ExitKind::Clean,
13287                        code: Some(0),
13288                        signal: None,
13289                        at_ms: 1,
13290                    },
13291                    true,
13292                ))
13293        });
13294        let mut reached = [0];
13295        barrier.read_exact(&mut reached).unwrap();
13296        assert_eq!(&reached, b"R");
13297        let (before_state, before_alive) = {
13298            let state = lock_snapshot(&snapshot).unwrap();
13299            (state.state, state.process_alive)
13300        };
13301        let before_history = history.lock().unwrap().snapshot();
13302        barrier.write_all(b"X").unwrap();
13303        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13304        assert!(before_history.entries.is_empty());
13305        assert_ne!(
13306            before_state,
13307            ModuleState::Failed,
13308            "Failed was visible before its terminal record could be written"
13309        );
13310        assert!(before_alive);
13311        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13312        let history = history.lock().unwrap().snapshot();
13313        assert_eq!(history.entries.len(), 1);
13314        assert_eq!(history.entries[0].exit_code, Some(0));
13315        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13316    }
13317
13318    #[test]
13319    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13320        use std::io::{Read, Write};
13321
13322        let supervisor = Supervisor::new_for_test(
13323            Arc::new(Registry::default()),
13324            RestartPolicy::new(3, Duration::ZERO),
13325        );
13326        let mut runtime = supervisor.runtime_config();
13327        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13328        let snapshot = module.inner.snapshot.clone();
13329        lock_snapshot(&snapshot).unwrap().process_alive = true;
13330        runtime.terminal_ring = module.inner.terminal_ring.clone();
13331        let spec = windowed_crash_spec(module.module_id());
13332        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13333        let reaped_snapshot = snapshot.clone();
13334        let reap = std::thread::spawn(move || {
13335            tokio::runtime::Builder::new_current_thread()
13336                .enable_all()
13337                .build()
13338                .unwrap()
13339                .block_on(on_child_exit(
13340                    &spec,
13341                    runtime.restart_policy,
13342                    &supervisor.registry,
13343                    &reaped_snapshot,
13344                    &runtime.terminal_ring,
13345                    &runtime.spawn_events,
13346                    &runtime.child_roster,
13347                    ExitReport {
13348                        kind: ExitKind::Crash,
13349                        code: Some(2),
13350                        signal: None,
13351                        at_ms: 1,
13352                    },
13353                    true,
13354                ))
13355        });
13356
13357        // Hold the crash handler between its budget decision and publication.
13358        // Connection cleanup must still see the recovery that this crash chose,
13359        // even when that recovery consumes the last available restart slot.
13360        let mut reached = [0];
13361        barrier.read_exact(&mut reached).unwrap();
13362        assert_eq!(&reached, b"R");
13363        let before_status = module.status().unwrap();
13364        let before_history = module.terminal_history();
13365        let before_recovers = module.will_recover_after_connection_loss().unwrap();
13366        barrier.write_all(b"X").unwrap();
13367        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13368
13369        assert_eq!(before_status.state, ModuleState::Running);
13370        assert!(before_status.process_alive);
13371        assert_eq!(before_status.restart_count, 3);
13372        assert!(before_history.entries.is_empty());
13373        assert!(
13374            before_recovers,
13375            "connection cleanup would report a terminal close for a scheduled crash restart"
13376        );
13377        let restarting = module.status().unwrap();
13378        assert_eq!(restarting.state, ModuleState::Restarting);
13379        assert_eq!(restarting.restart_count, 3);
13380        assert!(module.will_recover_after_connection_loss().unwrap());
13381        let history = module.terminal_history();
13382        assert_eq!(history.entries.len(), 1);
13383        assert_eq!(
13384            history.entries[0].disposition,
13385            TerminalDisposition::Restarting
13386        );
13387
13388        // Once the replacement is running, its next crash must be judged
13389        // against the exhausted budget rather than the previous recovery.
13390        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13391            state.state = ModuleState::Running;
13392            state.respawn_pending = false;
13393        })
13394        .unwrap();
13395        assert!(!module.will_recover_after_connection_loss().unwrap());
13396        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13397            state.enabled = false;
13398            state.respawn_pending = true;
13399        })
13400        .unwrap();
13401        assert!(!module.will_recover_after_connection_loss().unwrap());
13402    }
13403
13404    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13405        use std::net::{TcpListener, TcpStream};
13406        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13407        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13408        let (barrier, _) = listener.accept().unwrap();
13409        for stream in [&writer, &barrier] {
13410            stream
13411                .set_read_timeout(Some(Duration::from_secs(10)))
13412                .unwrap();
13413            stream
13414                .set_write_timeout(Some(Duration::from_secs(10)))
13415                .unwrap();
13416        }
13417        *events.1.lock().unwrap() = Some(writer);
13418        barrier
13419    }
13420
13421    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13422        use std::io::{Read, Write};
13423        let barrier = events.1.lock().unwrap().take();
13424        if let Some(mut barrier) = barrier {
13425            barrier.write_all(b"R").unwrap();
13426            let mut release = [0];
13427            barrier.read_exact(&mut release).unwrap();
13428            assert_eq!(&release, b"X");
13429        }
13430    }
13431
13432    #[test]
13433    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13434        use std::io::{Read, Write};
13435
13436        let supervisor = Supervisor::new_for_test(
13437            Arc::new(Registry::default()),
13438            RestartPolicy::new(0, Duration::ZERO),
13439        );
13440        let mut runtime = supervisor.runtime_config();
13441        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13442        let snapshot = module.inner.snapshot.clone();
13443        runtime.terminal_ring = module.inner.terminal_ring.clone();
13444        let spec = ModuleSpec {
13445            module_id: module.module_id().into(),
13446            program: fake_aft_stub_path(),
13447            protocol: ModuleProtocol::None,
13448            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13449            ..windowed_crash_spec(module.module_id())
13450        };
13451        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13452        let reap = std::thread::spawn(move || {
13453            tokio::runtime::Builder::new_current_thread()
13454                .enable_all()
13455                .build()
13456                .unwrap()
13457                .block_on(async {
13458                    let mut child =
13459                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13460                    health_restart_child(
13461                        &spec,
13462                        &runtime,
13463                        &supervisor.registry,
13464                        &supervisor.process_liveness,
13465                        &snapshot,
13466                        &mut child,
13467                        SupervisorHealthStatus::Failing,
13468                        Some("health probe did not answer"),
13469                        1,
13470                    )
13471                    .await
13472                    .unwrap();
13473                    assert!(child.is_none());
13474                });
13475        });
13476        // The loopback barrier acknowledges entry into terminal recording by
13477        // health_restart_child's drain, then waits for the test's release byte.
13478        // Status and history are observed while that writer is held, not after
13479        // an elapsed-time guess about when the health action has completed.
13480        let mut reached = [0];
13481        barrier.read_exact(&mut reached).unwrap();
13482        assert_eq!(&reached, b"R");
13483        let before_status = module.status().unwrap();
13484        let before_history = module.terminal_history();
13485        barrier.write_all(b"X").unwrap();
13486        reap.join().unwrap();
13487        assert!(before_history.entries.is_empty());
13488        assert_ne!(
13489            before_status.state,
13490            ModuleState::Failed,
13491            "Failed was visible before its terminal record could be written"
13492        );
13493        assert!(
13494            before_status.process_alive,
13495            "process death was visible before its terminal record could be written"
13496        );
13497        let failed = module.status().unwrap();
13498        assert_eq!(failed.state, ModuleState::Failed);
13499        assert!(!failed.process_alive);
13500        let history = module.terminal_history();
13501        assert_eq!(history.entries.len(), 1);
13502        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13503        assert_eq!(
13504            history.entries[0].disposition_detail.as_deref(),
13505            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13506        );
13507    }
13508
13509    #[test]
13510    fn observed_exits_publish_process_death_after_terminal_history() {
13511        use std::io::{Read, Write};
13512        for (kind, enabled, shutdown, wait_error, expected_state) in [
13513            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13514            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13515            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13516            (
13517                ExitKind::DeliberateSeverance,
13518                true,
13519                false,
13520                false,
13521                ModuleState::Restarting,
13522            ),
13523            #[cfg(unix)]
13524            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13525            (ExitKind::Crash, true, false, true, ModuleState::Failed),
13526        ] {
13527            let supervisor = Supervisor::new_for_test(
13528                Arc::new(Registry::default()),
13529                RestartPolicy::new(1, Duration::ZERO),
13530            );
13531            let runtime = supervisor.runtime_config();
13532            #[cfg(unix)]
13533            if shutdown {
13534                runtime.child_roster.close();
13535            }
13536            #[cfg(not(unix))]
13537            let _ = shutdown;
13538            let spec = windowed_crash_spec("exit-publication-order");
13539            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13540                ModuleState::Running,
13541                enabled,
13542            )));
13543            lock_snapshot(&snapshot).unwrap().process_alive = true;
13544            let reaped_snapshot = snapshot.clone();
13545            let history = runtime.terminal_ring.clone();
13546            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13547            let reap = std::thread::spawn(move || {
13548                if wait_error {
13549                    super::fail_after_wait_error(
13550                        &spec.module_id,
13551                        &reaped_snapshot,
13552                        &runtime.terminal_ring,
13553                        &runtime.spawn_events,
13554                        "wait failed".into(),
13555                        &runtime.failure_context(true),
13556                    );
13557                } else {
13558                    tokio::runtime::Builder::new_current_thread()
13559                        .enable_all()
13560                        .build()
13561                        .unwrap()
13562                        .block_on(on_child_exit(
13563                            &spec,
13564                            runtime.restart_policy,
13565                            &supervisor.registry,
13566                            &reaped_snapshot,
13567                            &runtime.terminal_ring,
13568                            &runtime.spawn_events,
13569                            &runtime.child_roster,
13570                            ExitReport {
13571                                kind,
13572                                code: Some(0),
13573                                signal: None,
13574                                at_ms: 1,
13575                            },
13576                            true,
13577                        ));
13578                }
13579            });
13580            let mut reached = [0];
13581            barrier.read_exact(&mut reached).unwrap();
13582            assert_eq!(&reached, b"R");
13583            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13584            let before_history = history.lock().unwrap().snapshot();
13585            barrier.write_all(b"X").unwrap();
13586            reap.join().unwrap();
13587            assert!(
13588                before_alive,
13589                "{kind:?}: death was published before recording"
13590            );
13591            assert!(before_history.entries.is_empty());
13592            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13593            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13594            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13595        }
13596    }
13597
13598    #[tokio::test]
13599    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13600        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13601        let path = dir.join("terminals.jsonl");
13602        std::fs::create_dir(&path).unwrap();
13603        let supervisor = Supervisor::new_for_test(
13604            Arc::new(Registry::default()),
13605            RestartPolicy::new(0, Duration::ZERO),
13606        )
13607        .with_terminal_journal(path, "health-budget-journal-failure".into());
13608        let runtime = supervisor.runtime_config();
13609        let spec = ModuleSpec {
13610            program: fake_aft_stub_path(),
13611            protocol: ModuleProtocol::None,
13612            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13613            ..windowed_crash_spec("health-budget-journal-failure")
13614        };
13615        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13616        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13617        health_restart_child(
13618            &spec,
13619            &runtime,
13620            &supervisor.registry,
13621            &supervisor.process_liveness,
13622            &snapshot,
13623            &mut child,
13624            SupervisorHealthStatus::Failing,
13625            None,
13626            1,
13627        )
13628        .await
13629        .unwrap();
13630        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13631        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13632        let history = runtime
13633            .terminal_ring
13634            .lock()
13635            .unwrap()
13636            .durable_history(&spec.module_id);
13637        assert!(history.journal_write_failures > 0);
13638        assert_eq!(history.entries.len(), 1);
13639        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13640    }
13641
13642    #[test]
13643    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13644        let supervisor = Supervisor::new_for_test(
13645            Arc::new(Registry::default()),
13646            RestartPolicy::new(0, Duration::ZERO),
13647        );
13648        let runtime = supervisor.runtime_config();
13649        let spec = ModuleSpec {
13650            program: fake_aft_stub_path(),
13651            protocol: ModuleProtocol::None,
13652            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13653            ..windowed_crash_spec("health-budget-recording-panic")
13654        };
13655        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13656        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13657            tokio::runtime::Builder::new_current_thread()
13658                .enable_all()
13659                .build()
13660                .unwrap()
13661                .block_on(async {
13662                    let mut child =
13663                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13664                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13665                    health_restart_child(
13666                        &spec,
13667                        &runtime,
13668                        &supervisor.registry,
13669                        &supervisor.process_liveness,
13670                        &snapshot,
13671                        &mut child,
13672                        SupervisorHealthStatus::Failing,
13673                        None,
13674                        1,
13675                    )
13676                    .await
13677                    .unwrap();
13678                });
13679        }));
13680        let panic = result.expect_err("recording must still unwind");
13681        assert_eq!(
13682            panic.downcast_ref::<String>().map(String::as_str),
13683            Some("spawn event sequence exhausted")
13684        );
13685        let state = lock_snapshot(&snapshot).unwrap();
13686        assert_eq!(state.state, ModuleState::Failed);
13687        assert!(!state.process_alive);
13688        assert!(state.last_exit.is_some());
13689    }
13690
13691    #[tokio::test]
13692    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13693        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13694        let path = dir.join("terminals.jsonl");
13695        std::fs::create_dir(&path).unwrap();
13696        let supervisor = Supervisor::new_for_test(
13697            Arc::new(Registry::default()),
13698            RestartPolicy::new(0, Duration::ZERO),
13699        )
13700        .with_terminal_journal(path, "budget-journal-failure".into());
13701        let runtime = supervisor.runtime_config();
13702        let spec = windowed_crash_spec("budget-journal-failure");
13703        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13704        assert!(matches!(
13705            on_child_exit(
13706                &spec,
13707                runtime.restart_policy,
13708                &supervisor.registry,
13709                &snapshot,
13710                &runtime.terminal_ring,
13711                &runtime.spawn_events,
13712                &runtime.child_roster,
13713                crash_exit_report(1),
13714                true,
13715            )
13716            .await,
13717            NextAction::Stop { .. }
13718        ));
13719        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13720        let history = runtime
13721            .terminal_ring
13722            .lock()
13723            .unwrap()
13724            .durable_history(&spec.module_id);
13725        assert!(history.journal_write_failures > 0);
13726        assert_eq!(history.entries.len(), 1);
13727        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13728    }
13729
13730    #[test]
13731    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13732        let supervisor = Supervisor::new_for_test(
13733            Arc::new(Registry::default()),
13734            RestartPolicy::new(0, Duration::ZERO),
13735        );
13736        let runtime = supervisor.runtime_config();
13737        let spec = windowed_crash_spec("budget-recording-panic");
13738        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13739        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13740        // Exhausting the event sequence makes emit_exited panic before the
13741        // terminal write, exercising publication on the recording unwind.
13742        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13743        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13744            tokio::runtime::Builder::new_current_thread()
13745                .enable_all()
13746                .build()
13747                .unwrap()
13748                .block_on(on_child_exit(
13749                    &spec,
13750                    runtime.restart_policy,
13751                    &supervisor.registry,
13752                    &snapshot,
13753                    &runtime.terminal_ring,
13754                    &runtime.spawn_events,
13755                    &runtime.child_roster,
13756                    crash_exit_report(1),
13757                    true,
13758                ))
13759        }));
13760        let panic = result.err().expect("recording must still unwind");
13761        assert_eq!(
13762            panic.downcast_ref::<String>().map(String::as_str),
13763            Some("spawn event sequence exhausted")
13764        );
13765        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13766    }
13767
13768    /// A stop the supervisor itself requests still stops a protocol-none
13769    /// module, even though the child answers the SIGTERM with exit 0.
13770    #[cfg(unix)]
13771    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13772    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13773        for disable in [false, true] {
13774            let label = if disable {
13775                "none-requested-disable"
13776            } else {
13777                "none-requested-stop"
13778            };
13779            let dir = cortexkit_test_support::ScratchDir::new(label);
13780            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13781            let supervisor = Supervisor::new_for_test(
13782                Arc::new(Registry::default()),
13783                RestartPolicy::new(3, Duration::ZERO),
13784            );
13785            let module = supervisor.spawn(spec).unwrap();
13786            wait_for_file(&ready).await;
13787
13788            if disable {
13789                module.set_enabled(false).await.unwrap();
13790            } else {
13791                module.stop().await.unwrap();
13792            }
13793            assert!(
13794                marker.exists(),
13795                "{label}: the child must have left through its SIGTERM handler with exit 0"
13796            );
13797
13798            // Long enough for a zero-backoff respawn to have happened if the
13799            // exit had been treated as a crash.
13800            sleep(Duration::from_millis(500)).await;
13801            let status = module.status().unwrap();
13802            let expected = if disable {
13803                ModuleState::Disabled
13804            } else {
13805                ModuleState::Stopped
13806            };
13807            assert_eq!(status.state, expected, "{label}");
13808            assert_eq!(
13809                status.spawn_generation, 1,
13810                "{label}: respawned after a requested stop"
13811            );
13812            let history = module.terminal_history();
13813            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13814            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13815            assert_ne!(
13816                history.entries[0].disposition,
13817                TerminalDisposition::Restarting,
13818                "{label}"
13819            );
13820        }
13821    }
13822
13823    /// A subc-wire module that exits 0 on its own is still a stop: the
13824    /// protocol-none rule must not reach it.
13825    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13826    async fn subc_wire_clean_exit_is_still_a_stop() {
13827        let supervisor = Supervisor::new_for_test(
13828            Arc::new(Registry::default()),
13829            RestartPolicy::new(3, Duration::ZERO),
13830        );
13831        let module = supervisor
13832            .spawn(ModuleSpec {
13833                module_id: "wire-clean-exit".to_string(),
13834                program: fake_aft_stub_path(),
13835                args: Vec::new(),
13836                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13837                reserved: false,
13838                reserved_prefixes: Vec::new(),
13839                protocol: ModuleProtocol::Subc,
13840                overlap: Default::default(),
13841                carry_containment: false,
13842            })
13843            .unwrap();
13844
13845        let deadline = Instant::now() + Duration::from_secs(10);
13846        while module.terminal_history().entries.is_empty() {
13847            assert!(Instant::now() < deadline, "module never exited");
13848            sleep(Duration::from_millis(10)).await;
13849        }
13850        // Long enough for a zero-backoff respawn to have happened.
13851        sleep(Duration::from_millis(500)).await;
13852        let status = module.status().unwrap();
13853        assert_eq!(status.state, ModuleState::Stopped);
13854        assert_eq!(status.spawn_generation, 1);
13855        let history = module.terminal_history();
13856        assert_eq!(history.entries.len(), 1, "{history:?}");
13857        assert_eq!(history.entries[0].exit_code, Some(0));
13858        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13859    }
13860
13861    #[cfg(unix)]
13862    #[tokio::test]
13863    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13864        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13865        let record = dir.join("live-children.json");
13866        let supervisor = Supervisor::new_for_test(
13867            Arc::new(Registry::default()),
13868            RestartPolicy::new(0, Duration::ZERO),
13869        );
13870        let mut runtime = supervisor.runtime_config();
13871        runtime.child_roster.record_to(record.clone());
13872        let gate = Arc::new(super::ReloadExitRecordGate::default());
13873        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13874        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13875        let spec = ModuleSpec {
13876            module_id: "reload-exit-roster".into(),
13877            program: fake_aft_stub_path(),
13878            args: Vec::new(),
13879            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13880            reserved: false,
13881            reserved_prefixes: Vec::new(),
13882            protocol: ModuleProtocol::Subc,
13883            overlap: Default::default(),
13884            carry_containment: false,
13885        };
13886        let mut child = None;
13887        let reload = super::finish_reload_child(
13888            &spec,
13889            &runtime,
13890            &supervisor.registry,
13891            &supervisor.process_liveness,
13892            &snapshot,
13893            &mut child,
13894        );
13895        tokio::pin!(reload);
13896        tokio::select! {
13897            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13898            _ = gate.reached.notified() => {}
13899        }
13900        assert!(runtime
13901            .terminal_ring
13902            .lock()
13903            .unwrap()
13904            .snapshot()
13905            .entries
13906            .is_empty());
13907        assert_eq!(
13908            crate::live_children::read_record(&record).unwrap().len(),
13909            1,
13910            "shutdown must still wait for the reaped child until its terminal record exists"
13911        );
13912        runtime.child_roster.close();
13913        gate.resume.notify_one();
13914        assert!(reload.await.is_err());
13915        assert!(crate::live_children::read_record(&record)
13916            .unwrap()
13917            .is_empty());
13918        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13919        assert_eq!(history.entries.len(), 1);
13920        assert_eq!(
13921            history.entries[0].disposition,
13922            TerminalDisposition::DaemonShutdown
13923        );
13924    }
13925
13926    /// Each restart-producing arm has its own state transition. Keeping their
13927    /// lifetime count assertions adjacent prevents a later new arm from silently
13928    /// spending budget without recording the historical restart.
13929    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13930    async fn every_restart_increment_path_advances_lifetime_count() {
13931        let supervisor = Supervisor::new_for_test(
13932            Arc::new(Registry::default()),
13933            RestartPolicy::new(1, Duration::ZERO),
13934        );
13935        let runtime = supervisor.runtime_config();
13936        let spec = ModuleSpec {
13937            module_id: "lifetime-increment-path".to_string(),
13938            program: PathBuf::from("/unused/lifetime-increment-path"),
13939            args: Vec::new(),
13940            env: Vec::new(),
13941            reserved: false,
13942            reserved_prefixes: Vec::new(),
13943            protocol: ModuleProtocol::Subc,
13944            overlap: Default::default(),
13945            carry_containment: false,
13946        };
13947
13948        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13949        assert!(matches!(
13950            on_child_exit(
13951                &spec,
13952                runtime.restart_policy,
13953                &supervisor.registry,
13954                &crash_snapshot,
13955                &runtime.terminal_ring,
13956                &runtime.spawn_events,
13957                &runtime.child_roster,
13958                ExitReport {
13959                    kind: ExitKind::Crash,
13960                    code: Some(1),
13961                    signal: None,
13962                    at_ms: 1,
13963                },
13964                true,
13965            )
13966            .await,
13967            NextAction::Restart { schedule: _ }
13968        ));
13969        let (crash_restarts, crash_lifetime) = {
13970            let state = lock_snapshot(&crash_snapshot).unwrap();
13971            (state.crash_restarts.len(), state.lifetime_restarts)
13972        };
13973        assert_eq!(crash_restarts, 1);
13974        assert_eq!(crash_lifetime, 1);
13975
13976        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13977        let mut health_child = None;
13978        assert!(matches!(
13979            health_restart_child(
13980                &spec,
13981                &runtime,
13982                &supervisor.registry,
13983                &supervisor.process_liveness,
13984                &health_snapshot,
13985                &mut health_child,
13986                SupervisorHealthStatus::Failing,
13987                None,
13988                2,
13989            )
13990            .await,
13991            Ok(())
13992        ));
13993        assert!(health_child.is_none());
13994        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13995        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13996        let (health_restarts, health_lifetime) = {
13997            let state = lock_snapshot(&health_snapshot).unwrap();
13998            (state.crash_restarts.len(), state.lifetime_restarts)
13999        };
14000        assert_eq!(health_restarts, 1);
14001        assert_eq!(health_lifetime, 1);
14002
14003        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14004        let mut reload_child = None;
14005        assert!(matches!(
14006            handle_reload_spawn_failure(
14007                &spec,
14008                &runtime,
14009                &supervisor.process_liveness,
14010                &reload_snapshot,
14011                &mut reload_child,
14012                "forced reload spawn failure".to_string(),
14013            )
14014            .await,
14015            Err(SuperviseError::ReloadFailed { .. })
14016        ));
14017        let (reload_restarts, reload_lifetime) = {
14018            let state = lock_snapshot(&reload_snapshot).unwrap();
14019            (state.crash_restarts.len(), state.lifetime_restarts)
14020        };
14021        assert_eq!(reload_restarts, 1);
14022        assert_eq!(reload_lifetime, 1);
14023    }
14024
14025    #[tokio::test]
14026    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14027        let supervisor = Supervisor::new_for_test(
14028            Arc::new(Registry::default()),
14029            RestartPolicy::new(3, Duration::ZERO),
14030        );
14031        let runtime = supervisor.runtime_config();
14032        let spec = ModuleSpec {
14033            module_id: "deliberately-severed".to_string(),
14034            program: PathBuf::from("/unused/deliberately-severed"),
14035            args: Vec::new(),
14036            env: Vec::new(),
14037            reserved: false,
14038            reserved_prefixes: Vec::new(),
14039            protocol: ModuleProtocol::Subc,
14040            overlap: Default::default(),
14041            carry_containment: false,
14042        };
14043        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14044        let process = ProcessIdentity {
14045            pid: 41,
14046            start_time: 101,
14047        };
14048        record_deliberate_severance(&snapshot, process).unwrap();
14049        let exit_report = apply_deliberate_severance_marker(
14050            &snapshot,
14051            Some(process),
14052            ExitReport {
14053                kind: ExitKind::Crash,
14054                code: Some(1),
14055                signal: None,
14056                at_ms: 1,
14057            },
14058        );
14059        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14060
14061        assert!(matches!(
14062            on_child_exit(
14063                &spec,
14064                runtime.restart_policy,
14065                &supervisor.registry,
14066                &snapshot,
14067                &runtime.terminal_ring,
14068                &runtime.spawn_events,
14069                &runtime.child_roster,
14070                exit_report,
14071                true,
14072            )
14073            .await,
14074            NextAction::Restart { schedule: _ }
14075        ));
14076        let state = lock_snapshot(&snapshot).unwrap();
14077        assert_eq!(state.lifetime_restarts, 1);
14078        assert_eq!(state.crash_restarts.len(), 0);
14079    }
14080
14081    #[tokio::test]
14082    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14083        let supervisor = Supervisor::new_for_test(
14084            Arc::new(Registry::default()),
14085            RestartPolicy::new(3, Duration::ZERO),
14086        );
14087        let runtime = supervisor.runtime_config();
14088        let spec = ModuleSpec {
14089            module_id: "genuine-crash".to_string(),
14090            program: PathBuf::from("/unused/genuine-crash"),
14091            args: Vec::new(),
14092            env: Vec::new(),
14093            reserved: false,
14094            reserved_prefixes: Vec::new(),
14095            protocol: ModuleProtocol::Subc,
14096            overlap: Default::default(),
14097            carry_containment: false,
14098        };
14099        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14100
14101        assert!(matches!(
14102            on_child_exit(
14103                &spec,
14104                runtime.restart_policy,
14105                &supervisor.registry,
14106                &snapshot,
14107                &runtime.terminal_ring,
14108                &runtime.spawn_events,
14109                &runtime.child_roster,
14110                ExitReport {
14111                    kind: ExitKind::Crash,
14112                    code: Some(1),
14113                    signal: None,
14114                    at_ms: 1,
14115                },
14116                true,
14117            )
14118            .await,
14119            NextAction::Restart { schedule: _ }
14120        ));
14121        let state = lock_snapshot(&snapshot).unwrap();
14122        assert_eq!(state.lifetime_restarts, 1);
14123        assert_eq!(state.crash_restarts.len(), 1);
14124    }
14125
14126    fn crash_exit_report(at_ms: u64) -> ExitReport {
14127        ExitReport {
14128            kind: ExitKind::Crash,
14129            code: Some(1),
14130            signal: None,
14131            at_ms,
14132        }
14133    }
14134
14135    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14136        ModuleSpec {
14137            module_id: module_id.to_string(),
14138            program: PathBuf::from("/unused").join(module_id),
14139            args: Vec::new(),
14140            env: Vec::new(),
14141            reserved: false,
14142            reserved_prefixes: Vec::new(),
14143            protocol: ModuleProtocol::Subc,
14144            overlap: Default::default(),
14145            carry_containment: false,
14146        }
14147    }
14148
14149    /// A real crash loop still stops. Three crashes with nothing aging out spend
14150    /// a budget of two and the third respawn is refused, and both surfaces an
14151    /// operator has -- the log line and the retained terminal record -- name the
14152    /// window rather than only the cap, because `max_restarts=2` alone is what
14153    /// this budget used to mean.
14154    #[tokio::test]
14155    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14156        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14157        let supervisor = Supervisor::new_for_test(
14158            Arc::new(Registry::default()),
14159            RestartPolicy::new(2, Duration::ZERO),
14160        );
14161        let runtime = supervisor.runtime_config();
14162        let spec = windowed_crash_spec("crash-loop-in-window");
14163        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14164
14165        for attempt in 1..=2 {
14166            assert!(
14167                matches!(
14168                    on_child_exit(
14169                        &spec,
14170                        runtime.restart_policy,
14171                        &supervisor.registry,
14172                        &snapshot,
14173                        &runtime.terminal_ring,
14174                        &runtime.spawn_events,
14175                        &runtime.child_roster,
14176                        crash_exit_report(attempt),
14177                        true,
14178                    )
14179                    .await,
14180                    NextAction::Restart { schedule: _ }
14181                ),
14182                "crash {attempt} is inside the budget and must respawn"
14183            );
14184        }
14185
14186        assert!(matches!(
14187            on_child_exit(
14188                &spec,
14189                runtime.restart_policy,
14190                &supervisor.registry,
14191                &snapshot,
14192                &runtime.terminal_ring,
14193                &runtime.spawn_events,
14194                &runtime.child_roster,
14195                crash_exit_report(3),
14196                true,
14197            )
14198            .await,
14199            NextAction::Stop { .. }
14200        ));
14201
14202        {
14203            let state = lock_snapshot(&snapshot).unwrap();
14204            assert_eq!(state.state, ModuleState::Failed);
14205            assert_eq!(state.crash_restarts.len(), 2);
14206            assert_eq!(state.lifetime_restarts, 2);
14207        }
14208
14209        let history = runtime
14210            .terminal_ring
14211            .lock()
14212            .expect("terminal ring is not poisoned")
14213            .snapshot();
14214        let last = history
14215            .entries
14216            .last()
14217            .expect("the refused crash is retained");
14218        assert_eq!(last.disposition, TerminalDisposition::Failed);
14219        assert_eq!(
14220            last.disposition_detail.as_deref(),
14221            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14222        );
14223
14224        let captured = crate::router::test_log::captured_logs(&logs);
14225        assert!(
14226            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14227            "the stop must be logged with its window: {captured}"
14228        );
14229    }
14230
14231    /// The rate, stated as a test: three crashes where the first has aged past
14232    /// the window are two crashes as far as the budget is concerned, so the
14233    /// third respawn is allowed and the ring holds only the two recent ones.
14234    ///
14235    /// This is the case a lifetime counter got wrong -- and the case the daemon
14236    /// now hits routinely, since a module exits non-zero every time its
14237    /// connection to the daemon drops.
14238    #[tokio::test]
14239    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14240        let supervisor = Supervisor::new_for_test(
14241            Arc::new(Registry::default()),
14242            RestartPolicy::new(2, Duration::ZERO),
14243        );
14244        let runtime = supervisor.runtime_config();
14245        let spec = windowed_crash_spec("crash-across-windows");
14246        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14247
14248        for attempt in 1..=2 {
14249            assert!(matches!(
14250                on_child_exit(
14251                    &spec,
14252                    runtime.restart_policy,
14253                    &supervisor.registry,
14254                    &snapshot,
14255                    &runtime.terminal_ring,
14256                    &runtime.spawn_events,
14257                    &runtime.child_roster,
14258                    crash_exit_report(attempt),
14259                    true,
14260                )
14261                .await,
14262                NextAction::Restart { schedule: _ }
14263            ));
14264        }
14265
14266        // The oldest crash moves out of the window; nothing else about the
14267        // module changes.
14268        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14269            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14270        })
14271        .unwrap();
14272
14273        assert!(
14274            matches!(
14275                on_child_exit(
14276                    &spec,
14277                    runtime.restart_policy,
14278                    &supervisor.registry,
14279                    &snapshot,
14280                    &runtime.terminal_ring,
14281                    &runtime.spawn_events,
14282                    &runtime.child_roster,
14283                    crash_exit_report(3),
14284                    true,
14285                )
14286                .await,
14287                NextAction::Restart { schedule: _ }
14288            ),
14289            "a crash older than the window must not hold a budget slot"
14290        );
14291
14292        let state = lock_snapshot(&snapshot).unwrap();
14293        assert_eq!(state.state, ModuleState::Restarting);
14294        assert_eq!(
14295            state.crash_restarts.len(),
14296            2,
14297            "the aged instant is dropped and the new one takes its place"
14298        );
14299        assert_eq!(
14300            state.lifetime_restarts, 3,
14301            "the ledger counts every restart, including the ones the window forgot"
14302        );
14303    }
14304
14305    /// An operator restart hands the budget back whole, and the ledger keeps
14306    /// counting. Those are different questions -- "how close is this module to
14307    /// being stopped" and "how many times has it been replaced" -- and the
14308    /// operator action answers only the first.
14309    #[tokio::test]
14310    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14311        let supervisor = Supervisor::new_for_test(
14312            Arc::new(Registry::default()),
14313            RestartPolicy::new(2, Duration::ZERO),
14314        );
14315        let runtime = supervisor.runtime_config();
14316        let spec = windowed_crash_spec("operator-cleared-budget");
14317        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14318
14319        for attempt in 1..=2 {
14320            assert!(matches!(
14321                on_child_exit(
14322                    &spec,
14323                    runtime.restart_policy,
14324                    &supervisor.registry,
14325                    &snapshot,
14326                    &runtime.terminal_ring,
14327                    &runtime.spawn_events,
14328                    &runtime.child_roster,
14329                    crash_exit_report(attempt),
14330                    true,
14331                )
14332                .await,
14333                NextAction::Restart { schedule: _ }
14334            ));
14335        }
14336
14337        reset_restart_count(&snapshot, &spec.module_id).unwrap();
14338        {
14339            let state = lock_snapshot(&snapshot).unwrap();
14340            assert!(
14341                state.crash_restarts.is_empty(),
14342                "an operator restart returns the full budget"
14343            );
14344            assert_eq!(
14345                state.lifetime_restarts, 2,
14346                "clearing the budget must not unmake the crashes"
14347            );
14348        }
14349
14350        assert!(
14351            matches!(
14352                on_child_exit(
14353                    &spec,
14354                    runtime.restart_policy,
14355                    &supervisor.registry,
14356                    &snapshot,
14357                    &runtime.terminal_ring,
14358                    &runtime.spawn_events,
14359                    &runtime.child_roster,
14360                    crash_exit_report(3),
14361                    true,
14362                )
14363                .await,
14364                NextAction::Restart { schedule: _ }
14365            ),
14366            "the cleared budget must be spendable again"
14367        );
14368        let state = lock_snapshot(&snapshot).unwrap();
14369        assert_eq!(state.crash_restarts.len(), 1);
14370        assert_eq!(state.lifetime_restarts, 3);
14371    }
14372
14373    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14374    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14375        let severed = ProcessIdentity {
14376            pid: 41,
14377            start_time: 101,
14378        };
14379        let successor = ProcessIdentity {
14380            pid: 41,
14381            start_time: 202,
14382        };
14383        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14384        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14385            state.pid = Some(successor.pid);
14386            state.process_start_time = Some(successor.start_time);
14387        })
14388        .unwrap();
14389        assert!(!module.record_deliberate_severance(severed).unwrap());
14390
14391        let exit_report = apply_deliberate_severance_marker(
14392            &module.inner.snapshot,
14393            Some(successor),
14394            ExitReport {
14395                kind: ExitKind::Crash,
14396                code: Some(1),
14397                signal: None,
14398                at_ms: 1,
14399            },
14400        );
14401
14402        assert_eq!(exit_report.kind, ExitKind::Crash);
14403    }
14404
14405    #[tokio::test]
14406    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14407        let registry = Registry::default();
14408        let supervisor = Supervisor::new_for_test(
14409            Arc::new(Registry::default()),
14410            RestartPolicy::new(3, Duration::ZERO),
14411        );
14412        let runtime = supervisor.runtime_config();
14413        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14414        let spec = ModuleSpec {
14415            module_id: "drain-deliberate-severance".to_string(),
14416            program: fake_aft_stub_path(),
14417            args: Vec::new(),
14418            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14419            reserved: false,
14420            reserved_prefixes: Vec::new(),
14421            protocol: ModuleProtocol::Subc,
14422            overlap: Default::default(),
14423            carry_containment: false,
14424        };
14425        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14426        let process = ProcessIdentity {
14427            pid: 41,
14428            start_time: 101,
14429        };
14430        child.process_identity = Some(process);
14431        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14432            state.pid = Some(process.pid);
14433            state.process_start_time = Some(process.start_time);
14434        })
14435        .unwrap();
14436        record_deliberate_severance(&snapshot, process).unwrap();
14437
14438        drain_child_to_state(
14439            &spec.module_id,
14440            spec.protocol,
14441            // The child exits on its own; no signal may change the exit this
14442            // test classifies.
14443            StopNotice::SentOverConnection,
14444            &registry,
14445            None,
14446            &snapshot,
14447            &runtime.terminal_ring,
14448            &runtime.spawn_events,
14449            child,
14450            Duration::from_secs(1),
14451            ModuleState::Stopped,
14452            Some(false),
14453            &super::FailureContext::operator(),
14454        )
14455        .await
14456        .unwrap();
14457
14458        let state = lock_snapshot(&snapshot).unwrap();
14459        assert_eq!(
14460            state.last_exit.as_ref().map(|exit| exit.kind),
14461            Some(ExitKind::DeliberateSeverance)
14462        );
14463        assert_eq!(state.lifetime_restarts, 1);
14464        assert_eq!(state.crash_restarts.len(), 0);
14465        drop(state);
14466        let history = runtime.terminal_ring.lock().unwrap().snapshot();
14467        assert_eq!(
14468            history.entries[0].exit_kind,
14469            subc_control::TerminalExitKind::DeliberateSeverance
14470        );
14471    }
14472
14473    #[tokio::test]
14474    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14475        let registry = Registry::default();
14476        let supervisor = Supervisor::new_for_test(
14477            Arc::new(Registry::default()),
14478            RestartPolicy::new(3, Duration::ZERO),
14479        );
14480        let runtime = supervisor.runtime_config();
14481        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14482        let spec = ModuleSpec {
14483            module_id: "ordinary-drain".to_string(),
14484            program: fake_aft_stub_path(),
14485            args: Vec::new(),
14486            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14487            reserved: false,
14488            reserved_prefixes: Vec::new(),
14489            protocol: ModuleProtocol::Subc,
14490            overlap: Default::default(),
14491            carry_containment: false,
14492        };
14493        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14494
14495        drain_child_to_state(
14496            &spec.module_id,
14497            spec.protocol,
14498            // The child exits on its own; no signal may change the exit this
14499            // test classifies.
14500            StopNotice::SentOverConnection,
14501            &registry,
14502            None,
14503            &snapshot,
14504            &runtime.terminal_ring,
14505            &runtime.spawn_events,
14506            child,
14507            Duration::from_secs(1),
14508            ModuleState::Stopped,
14509            Some(false),
14510            &super::FailureContext::operator(),
14511        )
14512        .await
14513        .unwrap();
14514
14515        let state = lock_snapshot(&snapshot).unwrap();
14516        assert_eq!(
14517            state.last_exit.as_ref().map(|exit| exit.kind),
14518            Some(ExitKind::Crash)
14519        );
14520        assert_eq!(state.lifetime_restarts, 0);
14521        assert_eq!(state.crash_restarts.len(), 0);
14522    }
14523
14524    #[test]
14525    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14526        // The server's generic fatal-routing branch only knows that the
14527        // connection failed; it does not know that the daemon deliberately
14528        // initiated a process-killing severance. Keep this seam explicit so a
14529        // future connection error path cannot silently reintroduce the stale
14530        // exemption that mislabels a later genuine crash.
14531        assert!(!include_str!("server.rs")
14532            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14533    }
14534
14535    /// The `route.closed` `drained` value must be the quiescence wait's own
14536    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
14537    /// measurement at all and `false` is the one honest constant. This is the exact
14538    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
14539    /// on every return path, including the one that used to return early via `?`
14540    /// with `route.closing` already sent and no `route.closed` ever following.
14541    #[test]
14542    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14543        assert!(drained_after_quiescence_wait(&Ok(true)));
14544        assert!(!drained_after_quiescence_wait(&Ok(false)));
14545        assert!(!drained_after_quiescence_wait(&Err(
14546            SuperviseError::StatePoisoned { module_id: None }
14547        )));
14548    }
14549
14550    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
14551    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
14552    /// already reaped out-of-band) still leaves a terminal record rather than none
14553    /// at all. Triggering the real `wait()` I/O error from an integration test would
14554    /// need a genuine already-reaped-child race, which is OS-specific and not
14555    /// something this suite attempts elsewhere; this test instead verifies the
14556    /// record produced for that arm end-to-end through the real `TerminalRing`, and
14557    /// the call site itself is verified by inspection to sit in that exact arm.
14558    #[test]
14559    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14560        let ring = Arc::new(Mutex::new(TerminalRing::new(
14561            TerminalRingConfig::default(),
14562            0,
14563        )));
14564        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14565
14566        let snapshot = ring.lock().unwrap().snapshot();
14567        assert_eq!(snapshot.entries.len(), 1);
14568        let entry = &snapshot.entries[0];
14569        assert_eq!(entry.exit_code, None);
14570        assert_eq!(entry.exit_signal, None);
14571        assert_eq!(entry.disposition, TerminalDisposition::Failed);
14572    }
14573
14574    #[test]
14575    fn wait_error_exit_path_preserves_spawn_event_density() {
14576        let feed = super::SpawnEventFeed::default();
14577        feed.configure_incarnation("wait-error-density".to_string());
14578        feed.emit_spawned("wait-error", 41, 1);
14579        let ring = Arc::new(Mutex::new(TerminalRing::new(
14580            TerminalRingConfig::default(),
14581            0,
14582        )));
14583
14584        record_wait_error_terminal("wait-error", &ring, &feed);
14585        feed.emit_spawned("after-wait-error", 42, 2);
14586
14587        let state = feed.0.lock().unwrap();
14588        let sequences = state
14589            .events
14590            .iter()
14591            .map(|event| event.cursor.seq)
14592            .collect::<Vec<_>>();
14593        assert_eq!(sequences, vec![1, 2, 3]);
14594        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14595        assert_eq!(state.events[1].exit_code, None);
14596        assert_eq!(state.events[1].exit_signal, None);
14597    }
14598
14599    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
14600    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
14601    /// not a clean exit it never actually observed.
14602    #[test]
14603    fn wait_error_exit_report_is_classified_as_a_crash() {
14604        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14605    }
14606}
14607
14608#[cfg(all(test, unix))]
14609mod restart_ordering_tests {
14610    use super::*;
14611    use std::io::Write;
14612    use tokio::io::AsyncReadExt;
14613
14614    #[test]
14615    #[ignore = "child process fixture"]
14616    fn serving_child() {
14617        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14618            return;
14619        };
14620        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14621        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14622        for stream in listener.incoming() {
14623            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14624        }
14625    }
14626
14627    async fn restart_before_publication(requests: usize) {
14628        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14629        let address_file = home.path().join("address");
14630        let spec = ModuleSpec {
14631            module_id: "restart-serving-child".into(),
14632            program: std::env::current_exe().unwrap(),
14633            args: vec![
14634                "--exact".into(),
14635                "supervise::restart_ordering_tests::serving_child".into(),
14636                "--ignored".into(),
14637                "--nocapture".into(),
14638            ],
14639            env: vec![(
14640                "RESTART_TEST_ADDRESS_FILE".into(),
14641                address_file.display().to_string(),
14642            )],
14643            reserved: false,
14644            reserved_prefixes: vec![],
14645            protocol: ModuleProtocol::None,
14646            overlap: Default::default(),
14647            carry_containment: false,
14648        };
14649        let mut supervisor =
14650            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14651        let (spawned_tx, spawned_rx) = oneshot::channel();
14652        let spawned_tx = Mutex::new(Some(spawned_tx));
14653        let (release_tx, release_rx) = std::sync::mpsc::channel();
14654        let release_rx = Mutex::new(release_rx);
14655        // Hold publication, not the child: it must answer a real request before
14656        // the restart enters the queue, even though status still has generation 0.
14657        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14658            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14659            release_rx
14660                .lock()
14661                .unwrap()
14662                .recv_timeout(Duration::from_secs(20))
14663                .unwrap();
14664        })));
14665        let registered = supervisor
14666            .register_configured_with_health(
14667                spec.clone(),
14668                true,
14669                HealthConfig::default(),
14670                None,
14671                RestartPolicy::new(3, Duration::ZERO),
14672                DEFAULT_REGISTER_TIMEOUT,
14673            )
14674            .unwrap();
14675        let module = registered.module.clone();
14676        let startup = tokio::task::spawn_blocking(move || {
14677            supervisor.start_registered_configured(registered, None, true)
14678        });
14679        spawned_rx.await.unwrap();
14680        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14681            let address = loop {
14682                if let Ok(address) = std::fs::read_to_string(&address_file) {
14683                    if !address.is_empty() {
14684                        break address;
14685                    }
14686                }
14687                tokio::time::sleep(Duration::from_millis(1)).await;
14688            };
14689            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14690            let mut reply = String::new();
14691            stream.read_to_string(&mut reply).await.unwrap();
14692            reply.trim().parse::<u32>().unwrap()
14693        })
14694        .await
14695        .expect("child must serve before generation publication");
14696        assert_eq!(
14697            lock_snapshot(&module.inner.snapshot)
14698                .unwrap()
14699                .spawn_generation,
14700            0
14701        );
14702        let mut replies = Vec::new();
14703        for _ in 0..requests {
14704            let mut restart = Box::pin(module.restart(Some(0)));
14705            // Poll the public API through queue admission, leaving only its ACK
14706            // pending. The monitor cannot consume the command until released.
14707            std::future::poll_fn(|cx| {
14708                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14709                std::task::Poll::Ready(())
14710            })
14711            .await;
14712            replies.push(restart);
14713        }
14714        release_tx.send(()).unwrap();
14715        startup.await.unwrap().unwrap();
14716        for reply in replies {
14717            reply.await.unwrap();
14718        }
14719        // Restart schedules its replacement after completing the handler. Wait
14720        // for that separate actor turn before placing the command barrier.
14721        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14722            loop {
14723                if lock_snapshot(&module.inner.snapshot)
14724                    .unwrap()
14725                    .spawn_generation
14726                    > 1
14727                {
14728                    break;
14729                }
14730                tokio::time::sleep(Duration::from_millis(1)).await;
14731            }
14732        })
14733        .await;
14734        // Commands are handled serially. The update-spec reply proves every
14735        // restart handler has finished before we inspect the snapshot; restart's
14736        // own ACK only acknowledges initiation, not replacement completion.
14737        module.update_spec_for_test(spec).await.unwrap();
14738        let (generation, pid) = {
14739            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14740            (state.spawn_generation, state.pid)
14741        };
14742        module.stop().await.unwrap();
14743        replacement.expect("restart must publish a replacement for the already-serving child");
14744        assert_ne!(
14745            pid,
14746            Some(old_pid),
14747            "a serving child predating the restart must be replaced"
14748        );
14749        assert_eq!(generation, 2, "queued restarts must share one replacement");
14750    }
14751
14752    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14753    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14754        restart_before_publication(1).await;
14755    }
14756
14757    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14758    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14759        restart_before_publication(2).await;
14760    }
14761
14762    #[tokio::test]
14763    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14764        let supervisor =
14765            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14766        let registered = supervisor
14767            .register_configured_with_health(
14768                ModuleSpec {
14769                    module_id: "queued-before-replacement".into(),
14770                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14771                    // macOS a module whose running image is not the configured
14772                    // program is refused once its exec is confirmed (see
14773                    // `record_write_stall_tests::sleeping_spec`).
14774                    program: "/bin/sleep".into(),
14775                    args: vec!["600".into()],
14776                    env: vec![],
14777                    reserved: false,
14778                    reserved_prefixes: vec![],
14779                    protocol: ModuleProtocol::None,
14780                    overlap: Default::default(),
14781                    carry_containment: false,
14782                },
14783                true,
14784                HealthConfig::default(),
14785                None,
14786                RestartPolicy::default(),
14787                DEFAULT_REGISTER_TIMEOUT,
14788            )
14789            .unwrap();
14790        let RegisteredConfiguredModule {
14791            mut spec,
14792            mut runtime,
14793            module,
14794            mut commands,
14795        } = registered;
14796        let snapshot = &module.inner.snapshot;
14797        let mut restart = Box::pin(module.restart(Some(0)));
14798        std::future::poll_fn(|cx| {
14799            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14800            std::task::Poll::Ready(())
14801        })
14802        .await;
14803        // Drive the actor explicitly so a replacement is definitely spawned
14804        // after receipt, but before this queued command is dequeued.
14805        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14806        let replacement_pid = child.as_ref().unwrap().id();
14807        let command = commands.recv().await.unwrap();
14808        assert!(
14809            handle_supervisor_command(
14810                command,
14811                &mut spec,
14812                &mut runtime,
14813                &supervisor.registry,
14814                &supervisor.process_liveness,
14815                snapshot,
14816                &mut child,
14817                &mut commands,
14818                &mut VecDeque::new(),
14819            )
14820            .await
14821        );
14822        restart.await.unwrap();
14823        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14824        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14825        if let Some(mut child) = child {
14826            child.start_kill().unwrap();
14827            child.wait().await.unwrap();
14828        }
14829        assert_eq!(
14830            kept_pid, replacement_pid,
14831            "a newer replacement must survive"
14832        );
14833        assert!(
14834            !pending,
14835            "a satisfied restart must not schedule another spawn"
14836        );
14837    }
14838}
14839
14840#[cfg(all(test, unix))]
14841mod health_event_tests {
14842    use super::*;
14843    use cortexkit_test_support::ScratchDir;
14844    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14845
14846    struct Actor {
14847        module: SupervisedModule,
14848        registry: Arc<Registry>,
14849        forwarding: Arc<ForwardingTable>,
14850        spec: ModuleSpec,
14851        health: HealthConfig,
14852        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14853        roster: ChildRoster,
14854        _home: ScratchDir,
14855    }
14856
14857    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14858        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14859        // for an observed condition, rather than letting paused time auto-advance
14860        // deadlines or assuming a fixed number of yields finishes the work.
14861        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14862        let tick = Instant::now();
14863        loop {
14864            if let Some(value) = observe() {
14865                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14866                return value;
14867            }
14868            assert!(
14869                std::time::Instant::now() < deadline,
14870                "timed out waiting for {reason}"
14871            );
14872            tokio::task::yield_now().await;
14873        }
14874    }
14875
14876    impl Actor {
14877        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14878            Self::start_with_policy(
14879                protocol,
14880                cadence,
14881                DEFAULT_REGISTER_TIMEOUT,
14882                RestartPolicy::new(3, Duration::ZERO),
14883            )
14884            .await
14885        }
14886
14887        async fn start_with_policy(
14888            protocol: ModuleProtocol,
14889            cadence: Duration,
14890            register_timeout: Duration,
14891            policy: RestartPolicy,
14892        ) -> Self {
14893            let home = ScratchDir::new("health-events");
14894            let registry = Arc::new(Registry::default());
14895            let forwarding = Arc::new(ForwardingTable::default());
14896            let health = HealthConfig {
14897                cadence,
14898                deadline: Duration::from_secs(3600),
14899                ..HealthConfig::default()
14900            };
14901            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14902                .with_forwarding(forwarding.clone())
14903                .with_health_config(health.clone());
14904            let spec = ModuleSpec {
14905                module_id: "event-health-child".into(),
14906                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14907                // the privacy trampoline refuses a module whose running image
14908                // is not the configured program, and `start` below waits for
14909                // the process to be confirmed (`reported_pid`). The shell form
14910                // passed only while the image check ran before the shell had
14911                // replaced itself with `sleep` (see
14912                // `record_write_stall_tests::sleeping_spec`).
14913                program: PathBuf::from("/bin/sleep"),
14914                // The process stays alive but has no real bus peer. Registrations
14915                // below exercise the same registry writes as accepted HELLOs.
14916                args: vec!["600".into()],
14917                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14918                    .into_iter()
14919                    .map(|key| (key.into(), home.path().display().to_string()))
14920                    .collect(),
14921                reserved: false,
14922                reserved_prefixes: vec![],
14923                protocol,
14924                overlap: Default::default(),
14925                carry_containment: false,
14926            };
14927            let module = supervisor
14928                .supervise_configured_requested(
14929                    spec.clone(),
14930                    true,
14931                    health.clone(),
14932                    Some(0),
14933                    policy,
14934                    register_timeout,
14935                    None,
14936                )
14937                .unwrap();
14938            let (_, rx) = mpsc::channel(8);
14939            let actor = Self {
14940                module,
14941                registry,
14942                forwarding,
14943                spec,
14944                health,
14945                rx,
14946                roster: supervisor.child_roster.clone(),
14947                _home: home,
14948            };
14949            actor
14950                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14951                .await;
14952            actor
14953        }
14954
14955        async fn expire_registration_wait(&self, remaining: Duration) {
14956            tokio::time::advance(remaining).await;
14957            wait_for("registration timeout TERM grace", || {
14958                lock_snapshot(&self.module.inner.snapshot)
14959                    .unwrap()
14960                    .actor_select
14961                    .is_none()
14962                    .then_some(())
14963            })
14964            .await;
14965            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14966        }
14967
14968        fn turns(&self) -> u64 {
14969            lock_snapshot(&self.module.inner.snapshot)
14970                .unwrap()
14971                .actor_turns
14972        }
14973
14974        async fn wait_for_select(
14975            &self,
14976            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14977        ) -> ActorSelectCheckpoint {
14978            wait_for("actor select checkpoint", || {
14979                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14980                state.actor_select.filter(|checkpoint| {
14981                    checkpoint.generation == state.spawn_generation
14982                        && state.state == ModuleState::Running
14983                        && state.process_alive
14984                        && state.reported_pid().is_some()
14985                        && expected(checkpoint)
14986                })
14987            })
14988            .await
14989        }
14990
14991        fn hello(&mut self, connection: u64, health: bool) {
14992            let manifest =
14993                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14994                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14995                    .build();
14996            let connection = ConnectionId::new(connection);
14997            let (tx, rx) = mpsc::channel(8);
14998            self.rx = rx;
14999            self.forwarding
15000                .register_module_connection(
15001                    connection,
15002                    self.spec.module_id.clone(),
15003                    subc_protocol::PROTOCOL_VERSION,
15004                    Concurrency::ModuleManaged,
15005                    FrameSink::new(tx),
15006                )
15007                .unwrap();
15008            self.registry
15009                .register_with_control_ops(
15010                    manifest,
15011                    subc_protocol::PROTOCOL_VERSION,
15012                    connection,
15013                    if health {
15014                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15015                    } else {
15016                        vec![]
15017                    },
15018                )
15019                .unwrap();
15020        }
15021
15022        async fn probe(&mut self) -> crate::router::OutboundFrame {
15023            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15024                Ok(frame) => Some(frame),
15025                Err(mpsc::error::TryRecvError::Empty) => None,
15026                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15027            })
15028            .await;
15029            let body: Value = serde_json::from_slice(&frame.body).unwrap();
15030            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15031            frame
15032        }
15033
15034        fn no_probe(&mut self) {
15035            assert!(matches!(
15036                self.rx.try_recv(),
15037                Err(mpsc::error::TryRecvError::Empty)
15038            ));
15039        }
15040
15041        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15042            self.forwarding
15043                .complete_module_control_rpc(
15044                    ConnectionId::new(connection),
15045                    frame.header.corr,
15046                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15047                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15048                        status: HealthStatus::Ok,
15049                        detail: None,
15050                        metrics: None,
15051                    }),
15052                )
15053                .unwrap();
15054        }
15055    }
15056
15057    #[tokio::test(start_paused = true)]
15058    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15059        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15060        let initial = actor.module.status().unwrap();
15061        tokio::time::advance(Duration::from_secs(119)).await;
15062        tokio::task::yield_now().await;
15063        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15064        assert!(actor.module.terminal_history().entries.is_empty());
15065        actor.expire_registration_wait(Duration::from_secs(1)).await;
15066        actor
15067            .wait_for_select(|at| at.generation > initial.spawn_generation)
15068            .await;
15069        let status = actor.module.status().unwrap();
15070        assert_ne!(status.pid, initial.pid);
15071        assert!(!cortexkit_test_support::process_alive(
15072            initial.pid.unwrap() as i32
15073        ));
15074        assert_eq!(status.restart_count, 1);
15075        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15076        let entries = actor.module.terminal_history().entries;
15077        assert_eq!(entries.len(), 1);
15078        assert_eq!(
15079            entries[0].disposition_detail.as_deref(),
15080            Some("registration_timeout")
15081        );
15082        actor.module.stop().await.unwrap();
15083    }
15084
15085    #[tokio::test(start_paused = true)]
15086    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15087        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15088        let pid = actor.module.status().unwrap().pid;
15089        tokio::time::advance(Duration::from_secs(119)).await;
15090        actor.hello(1, false);
15091        let registered = actor
15092            .wait_for_select(|at| at.registered_connection.is_some())
15093            .await;
15094        assert_eq!(
15095            registered.wake_after, None,
15096            "HELLO must cancel the deadline"
15097        );
15098        tokio::time::advance(Duration::from_secs(300)).await;
15099        tokio::task::yield_now().await;
15100        assert_eq!(actor.module.status().unwrap().pid, pid);
15101        assert!(actor.module.terminal_history().entries.is_empty());
15102        // A disconnect after the first HELLO must not re-arm a startup timer.
15103        actor
15104            .registry
15105            .deregister_connection(ConnectionId::new(1))
15106            .unwrap();
15107        actor
15108            .wait_for_select(|at| at.registered_connection.is_none())
15109            .await;
15110        tokio::time::advance(Duration::from_secs(300)).await;
15111        tokio::task::yield_now().await;
15112        assert_eq!(actor.module.status().unwrap().pid, pid);
15113        assert!(actor.module.terminal_history().entries.is_empty());
15114        actor.module.stop().await.unwrap();
15115
15116        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15117        let pid = transient.module.status().unwrap().pid;
15118        tokio::time::advance(Duration::from_secs(119)).await;
15119        let turns = transient.turns();
15120        // Both writes happen before the supervisor can refresh the registry.
15121        // An accepted HELLO must remain evidence after its connection is gone.
15122        transient.hello(1, false);
15123        transient
15124            .registry
15125            .deregister_connection(ConnectionId::new(1))
15126            .unwrap();
15127        let parked = transient.wait_for_select(|at| at.turn > turns).await;
15128        assert_eq!(
15129            parked.wake_after, None,
15130            "transient HELLO must cancel the deadline"
15131        );
15132        tokio::time::advance(Duration::from_secs(300)).await;
15133        tokio::task::yield_now().await;
15134        assert_eq!(transient.module.status().unwrap().pid, pid);
15135        assert!(transient.module.terminal_history().entries.is_empty());
15136        transient.module.stop().await.unwrap();
15137    }
15138
15139    #[tokio::test(start_paused = true)]
15140    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15141        let actor = Actor::start_with_policy(
15142            ModuleProtocol::Subc,
15143            Duration::from_secs(30),
15144            Duration::from_secs(10),
15145            RestartPolicy::new(1, Duration::ZERO),
15146        )
15147        .await;
15148        let generation = actor.module.status().unwrap().spawn_generation;
15149        actor
15150            .expire_registration_wait(Duration::from_secs(10))
15151            .await;
15152        actor.wait_for_select(|at| at.generation > generation).await;
15153        actor
15154            .expire_registration_wait(Duration::from_secs(10))
15155            .await;
15156        let retry = wait_for("failed retry schedule", || {
15157            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15158            (state.state == ModuleState::Failed)
15159                .then_some(state.failed_retry.pending)
15160                .flatten()
15161        })
15162        .await;
15163        assert_eq!(retry.delay, Duration::from_secs(600));
15164        assert_eq!(retry.attempt, 1);
15165        let entries = actor.module.terminal_history().entries;
15166        assert_eq!(entries.len(), 2);
15167        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15168        assert!(entries[1]
15169            .disposition_detail
15170            .as_deref()
15171            .unwrap()
15172            .contains("registration_timeout"));
15173        tokio::time::advance(Duration::from_secs(599)).await;
15174        tokio::task::yield_now().await;
15175        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15176        tokio::time::advance(Duration::from_secs(1)).await;
15177        actor
15178            .wait_for_select(|at| at.generation > generation + 1)
15179            .await;
15180        // The slow retry is another ordinary spawn, with its own HELLO budget.
15181        actor
15182            .expire_registration_wait(Duration::from_secs(10))
15183            .await;
15184        wait_for("slow retry registration timeout", || {
15185            (actor.module.terminal_history().entries.len() == 3).then_some(())
15186        })
15187        .await;
15188        actor.module.stop().await.unwrap();
15189    }
15190
15191    #[tokio::test(start_paused = true)]
15192    async fn registration_timeout_honours_ten_second_override() {
15193        let actor = Actor::start_with_policy(
15194            ModuleProtocol::Subc,
15195            Duration::from_secs(30),
15196            Duration::from_secs(10),
15197            RestartPolicy::new(3, Duration::ZERO),
15198        )
15199        .await;
15200        let generation = actor.module.status().unwrap().spawn_generation;
15201        tokio::time::advance(Duration::from_secs(9)).await;
15202        tokio::task::yield_now().await;
15203        assert!(actor.module.terminal_history().entries.is_empty());
15204        actor.expire_registration_wait(Duration::from_secs(1)).await;
15205        actor.wait_for_select(|at| at.generation > generation).await;
15206        assert_eq!(actor.module.terminal_history().entries.len(), 1);
15207        actor.module.stop().await.unwrap();
15208    }
15209
15210    #[tokio::test]
15211    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15212        use std::os::unix::process::{CommandExt, ExitStatusExt};
15213
15214        struct Helper(std::process::Child);
15215        impl Drop for Helper {
15216            fn drop(&mut self) {
15217                // This handle still owns the helper until wait returns. Never
15218                // clean up its old process group after the leader is reaped.
15219                if matches!(self.0.try_wait(), Ok(None)) {
15220                    let _ = self.0.kill();
15221                    let _ = self.0.wait();
15222                }
15223            }
15224        }
15225
15226        let home = ScratchDir::new("registration-timeout-helper");
15227        let supervisor = Supervisor::new_for_test(
15228            Arc::new(Registry::default()),
15229            RestartPolicy::new(0, Duration::ZERO),
15230        );
15231        let spec = ModuleSpec {
15232            module_id: "registration-timeout-group".into(),
15233            program: "/bin/sleep".into(),
15234            args: vec!["120".into()],
15235            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15236                .into_iter()
15237                .map(|key| (key.into(), home.path().display().to_string()))
15238                .collect(),
15239            reserved: false,
15240            reserved_prefixes: vec![],
15241            protocol: ModuleProtocol::Subc,
15242            overlap: Default::default(),
15243            carry_containment: false,
15244        };
15245        let module = supervisor
15246            .supervise_configured_requested(
15247                spec,
15248                true,
15249                HealthConfig::default(),
15250                None,
15251                RestartPolicy::new(0, Duration::ZERO),
15252                Duration::from_secs(10),
15253                None,
15254            )
15255            .unwrap();
15256        let leader = timeout(Duration::from_secs(5), async {
15257            loop {
15258                if let Some(pid) = module.status().unwrap().pid {
15259                    break pid;
15260                }
15261                sleep(Duration::from_millis(5)).await;
15262            }
15263        })
15264        .await
15265        .expect("supervised group leader must be reported before the deadline");
15266        assert!(leader > 1);
15267        let ready = home.join("helper.ready");
15268        // Own the helper separately so even a broken group kill can be cleaned
15269        // up by its own unreaped child handle, without signalling a stale pgid.
15270        let mut helper = Helper(
15271            std::process::Command::new("/bin/sh")
15272                .args([
15273                    "-c",
15274                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15275                    "helper",
15276                ])
15277                .arg(&ready)
15278                .process_group(i32::try_from(leader).unwrap())
15279                .stdout(std::process::Stdio::null())
15280                .stderr(std::process::Stdio::null())
15281                .spawn()
15282                .unwrap(),
15283        );
15284        timeout(Duration::from_secs(5), async {
15285            while !ready.exists() {
15286                assert!(helper.0.try_wait().unwrap().is_none());
15287                sleep(Duration::from_millis(5)).await;
15288            }
15289        })
15290        .await
15291        .expect("helper must install TERM ignore before the registration deadline");
15292        let helper_pid =
15293            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15294        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15295        sleep(Duration::from_millis(20)).await;
15296        assert!(
15297            helper.0.try_wait().unwrap().is_none(),
15298            "helper must ignore TERM"
15299        );
15300
15301        timeout(Duration::from_secs(20), async {
15302            while module.state().unwrap() != ModuleState::Failed {
15303                sleep(Duration::from_millis(5)).await;
15304            }
15305        })
15306        .await
15307        .expect("registration timeout must stop and reap the group leader");
15308        let helper_exit = timeout(Duration::from_secs(3), async {
15309            loop {
15310                if let Some(status) = helper.0.try_wait().unwrap() {
15311                    break status;
15312                }
15313                sleep(Duration::from_millis(5)).await;
15314            }
15315        })
15316        .await
15317        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15318        assert_eq!(helper_exit.signal(), Some(9));
15319        assert!(!cortexkit_test_support::process_alive(
15320            i32::try_from(leader).unwrap()
15321        ));
15322        assert_eq!(
15323            module.status().unwrap().last_exit.unwrap().kind,
15324            ExitKind::Crash
15325        );
15326        assert!(module.terminal_history().entries[0]
15327            .disposition_detail
15328            .as_deref()
15329            .unwrap()
15330            .contains("registration_timeout"));
15331        module.stop().await.unwrap();
15332    }
15333
15334    #[tokio::test(start_paused = true)]
15335    async fn registration_timeout_does_not_count_an_earlier_exit() {
15336        let actor = Actor::start_with_policy(
15337            ModuleProtocol::Subc,
15338            Duration::from_secs(30),
15339            DEFAULT_REGISTER_TIMEOUT,
15340            RestartPolicy::new(0, Duration::ZERO),
15341        )
15342        .await;
15343        let pid = actor.module.status().unwrap().pid.unwrap();
15344        rustix::process::kill_process(
15345            rustix::process::Pid::from_raw(pid as i32).unwrap(),
15346            rustix::process::Signal::TERM,
15347        )
15348        .unwrap();
15349        wait_for("early exit", || {
15350            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15351        })
15352        .await;
15353        tokio::time::advance(Duration::from_secs(121)).await;
15354        tokio::task::yield_now().await;
15355        let entries = actor.module.terminal_history().entries;
15356        assert_eq!(entries.len(), 1);
15357        assert!(!entries[0]
15358            .disposition_detail
15359            .as_deref()
15360            .unwrap()
15361            .contains("registration_timeout"));
15362        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15363        actor.module.stop().await.unwrap();
15364    }
15365
15366    #[tokio::test(start_paused = true)]
15367    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15368        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15369        let pid = actor.module.status().unwrap().pid;
15370        tokio::time::advance(Duration::from_secs(119)).await;
15371        actor.roster.close();
15372        let turns = actor.turns();
15373        tokio::time::advance(Duration::from_secs(300)).await;
15374        let parked = actor.wait_for_select(|at| at.turn > turns).await;
15375        assert_eq!(parked.wake_after, None);
15376        assert_eq!(actor.module.status().unwrap().pid, pid);
15377        assert!(actor.module.terminal_history().entries.is_empty());
15378        actor.module.stop().await.unwrap();
15379        let status = actor.module.status().unwrap();
15380        assert_eq!(status.restart_count, 0);
15381        assert_eq!(
15382            actor.module.terminal_history().entries[0].disposition,
15383            TerminalDisposition::DaemonShutdown
15384        );
15385    }
15386
15387    #[tokio::test(start_paused = true)]
15388    async fn no_health_children_park_for_sixty_seconds() {
15389        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15390        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15391        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15392        unadvertised.hello(1, false);
15393        // Include any one-off registration wake in the budget; receiving a
15394        // no-health HELLO must not turn a parked child into a polling child.
15395        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
15396        // a single 60 s jump would only observe one expired timer.
15397        for _ in 0..6000 {
15398            tokio::time::advance(Duration::from_millis(10)).await;
15399            tokio::task::yield_now().await;
15400        }
15401        for actor in [&none, &unregistered, &unadvertised] {
15402            assert!(
15403                actor.turns() <= 5,
15404                "no-health actor woke {} times",
15405                actor.turns()
15406            );
15407            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15408        }
15409    }
15410
15411    #[tokio::test(start_paused = true)]
15412    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15413        // With a zero cadence the first probe is due at once, so this measures
15414        // only how fast the registration notification starts health probing,
15415        // not the normal 30-33 s wait before a first probe (checked below).
15416        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15417        let tick = Instant::now();
15418        actor.hello(1, true);
15419        actor.probe().await;
15420        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15421    }
15422
15423    #[tokio::test(start_paused = true)]
15424    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15425        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15426        let hello_tick = Instant::now();
15427        actor.hello(1, true);
15428        let armed = actor
15429            .wait_for_select(|checkpoint| {
15430                checkpoint.registered_connection == Some(ConnectionId::new(1))
15431                    && checkpoint.next_probe_at.is_some()
15432            })
15433            .await;
15434        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15435        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15436        let first_deadline = hello_tick + delay;
15437        assert_eq!(armed.next_probe_at, Some(first_deadline));
15438        tokio::time::advance(Duration::from_secs(29)).await;
15439        let turns = actor.turns();
15440        actor
15441            .registry
15442            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15443            .unwrap();
15444        let updated = actor
15445            .wait_for_select(|checkpoint| checkpoint.turn > turns)
15446            .await;
15447        assert_eq!(
15448            updated.next_probe_at,
15449            Some(first_deadline),
15450            "catalog update must not reset cadence"
15451        );
15452        actor.no_probe();
15453        tokio::time::advance(first_deadline - Instant::now()).await;
15454        let frame = actor.probe().await;
15455        actor.answer(1, frame);
15456        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15457        let next_deadline = Instant::now() + next;
15458        let rearmed = actor
15459            .wait_for_select(|checkpoint| {
15460                checkpoint
15461                    .next_probe_at
15462                    .is_some_and(|deadline| deadline > first_deadline)
15463            })
15464            .await;
15465        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15466        tokio::time::advance(next - Duration::from_nanos(1)).await;
15467        actor.no_probe();
15468        tokio::time::advance(Duration::from_nanos(1)).await;
15469        actor.probe().await;
15470    }
15471
15472    #[tokio::test(start_paused = true)]
15473    async fn disconnect_disarms_and_reconnect_rearms_health() {
15474        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15475        actor.hello(1, true);
15476        actor
15477            .wait_for_select(|checkpoint| {
15478                checkpoint.registered_connection == Some(ConnectionId::new(1))
15479                    && checkpoint.next_probe_at.is_some()
15480            })
15481            .await;
15482        let turns = actor.turns();
15483        let tick = Instant::now();
15484        actor
15485            .registry
15486            .deregister_connection(ConnectionId::new(1))
15487            .unwrap();
15488        let disarmed = actor
15489            .wait_for_select(|checkpoint| {
15490                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15491            })
15492            .await;
15493        assert_eq!(disarmed.next_probe_at, None);
15494        assert_eq!(disarmed.wake_after, None);
15495        assert_eq!(
15496            Instant::now(),
15497            tick,
15498            "disconnect must disarm in the notification tick"
15499        );
15500        tokio::time::advance(Duration::from_secs(60)).await;
15501        actor.no_probe();
15502        let reconnect_tick = Instant::now();
15503        actor.hello(2, true);
15504        let rearmed = actor
15505            .wait_for_select(|checkpoint| {
15506                checkpoint.registered_connection == Some(ConnectionId::new(2))
15507                    && checkpoint.next_probe_at.is_some()
15508            })
15509            .await;
15510        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15511        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15512        tokio::time::advance(delay).await;
15513        actor.probe().await;
15514    }
15515
15516    #[tokio::test(start_paused = true)]
15517    async fn reregistration_without_health_disarms_the_old_schedule() {
15518        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15519        actor.hello(1, true);
15520        actor
15521            .wait_for_select(|checkpoint| {
15522                checkpoint.registered_connection == Some(ConnectionId::new(1))
15523                    && checkpoint.next_probe_at.is_some()
15524            })
15525            .await;
15526        actor
15527            .registry
15528            .deregister_connection(ConnectionId::new(1))
15529            .unwrap();
15530        actor.hello(2, false);
15531        let disarmed = actor
15532            .wait_for_select(|checkpoint| {
15533                checkpoint.registered_connection == Some(ConnectionId::new(2))
15534            })
15535            .await;
15536        assert_eq!(disarmed.next_probe_at, None);
15537        assert_eq!(disarmed.wake_after, None);
15538        tokio::time::advance(Duration::from_secs(60)).await;
15539        actor.no_probe();
15540    }
15541
15542    #[tokio::test(start_paused = true)]
15543    async fn restart_waits_for_its_own_hello_before_rearming_health() {
15544        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15545        actor.hello(1, true);
15546        let armed = actor
15547            .wait_for_select(|checkpoint| {
15548                checkpoint.registered_connection == Some(ConnectionId::new(1))
15549                    && checkpoint.next_probe_at.is_some()
15550            })
15551            .await;
15552        // Simulate the old process's connection closing before the restart
15553        // tears the process down. The replacement process has started but has
15554        // not sent its HELLO, so it is not registered.
15555        actor
15556            .registry
15557            .deregister_connection(ConnectionId::new(1))
15558            .unwrap();
15559        actor.module.restart(Some(0)).await.unwrap();
15560        let parked = actor
15561            .wait_for_select(|checkpoint| {
15562                checkpoint.generation > armed.generation
15563                    && checkpoint.registered_connection.is_none()
15564                    && checkpoint.next_probe_at.is_none()
15565            })
15566            .await;
15567        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15568        actor.rx.close();
15569        // Drain the stop notice the restart queued for the old process's connection,
15570        // so it isn't mistaken for traffic to the replacement.
15571        while actor.rx.try_recv().is_ok() {}
15572        let turns = actor.turns();
15573        tokio::time::advance(Duration::from_secs(60)).await;
15574        let after = actor.turns();
15575        eprintln!(
15576            "restart park turns: before={turns} after={after} delta={}",
15577            after - turns
15578        );
15579        assert_eq!(after, turns, "replacement must park before HELLO");
15580        let hello_tick = Instant::now();
15581        actor.hello(2, true);
15582        let rearmed = actor
15583            .wait_for_select(|checkpoint| {
15584                checkpoint.registered_connection == Some(ConnectionId::new(2))
15585                    && checkpoint.next_probe_at.is_some()
15586            })
15587            .await;
15588        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15589        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15590        tokio::time::advance(delay).await;
15591        actor.probe().await;
15592    }
15593
15594    #[tokio::test(start_paused = true)]
15595    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15596        use tokio::{
15597            io::{AsyncReadExt, AsyncWriteExt},
15598            net::TcpListener,
15599        };
15600        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15601        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15602        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15603        let (started_tx, mut started_rx) = mpsc::channel(8);
15604        let peer = tokio::spawn(async move {
15605            loop {
15606                let (mut socket, _) = listener.accept().await.unwrap();
15607                let mut request = [0; 1024];
15608                assert!(socket.read(&mut request).await.unwrap() > 0);
15609                started_tx.send(Instant::now()).await.unwrap();
15610                socket
15611                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15612                    .await
15613                    .unwrap();
15614            }
15615        });
15616        actor
15617            .module
15618            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15619            .await
15620            .unwrap();
15621        let first_tick = Instant::now();
15622        let first = actor
15623            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15624            .await;
15625        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15626        assert!(started_rx.try_recv().is_err());
15627
15628        // Restart just before the old deadline: reusing it would give the new
15629        // process only one second to start, rather than a full cadence.
15630        tokio::time::advance(
15631            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15632        )
15633        .await;
15634        actor.module.restart(Some(0)).await.unwrap();
15635        let spawn_tick = Instant::now();
15636        let replacement = actor
15637            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15638            .await;
15639        let deadline = replacement.next_probe_at.unwrap();
15640        assert!(
15641            deadline >= spawn_tick + actor.health.cadence,
15642            "a replacement must not inherit its predecessor's probe deadline"
15643        );
15644        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15645        assert!(
15646            started_rx.try_recv().is_err(),
15647            "no HTTP probe during startup grace"
15648        );
15649        tokio::time::advance(deadline - Instant::now()).await;
15650        let landed = wait_for("first replacement HTTP probe", || {
15651            started_rx.try_recv().ok()
15652        })
15653        .await;
15654        assert!(landed >= spawn_tick + actor.health.cadence);
15655        wait_for("healthy replacement", || {
15656            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15657                .then_some(())
15658        })
15659        .await;
15660        peer.abort();
15661    }
15662
15663    #[tokio::test(start_paused = true)]
15664    async fn rescan_adds_and_removes_http_health_without_polling() {
15665        use tokio::{
15666            io::{AsyncReadExt, AsyncWriteExt},
15667            net::TcpListener,
15668        };
15669        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15670        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15671        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15672        // One millisecond separates successive probes without a polling delay.
15673        actor.health.cadence = Duration::from_millis(1);
15674        let (started_tx, mut started_rx) = mpsc::channel(8);
15675        let peer = tokio::spawn(async move {
15676            loop {
15677                let (mut socket, _) = listener.accept().await.unwrap();
15678                let mut request = [0; 1024];
15679                assert!(socket.read(&mut request).await.unwrap() > 0);
15680                started_tx.send(()).await.unwrap();
15681                socket
15682                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15683                    .await
15684                    .unwrap();
15685            }
15686        });
15687        let tick = Instant::now();
15688        let turns = actor.turns();
15689        actor
15690            .module
15691            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15692            .await
15693            .unwrap();
15694        let armed = actor
15695            .wait_for_select(|checkpoint| {
15696                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15697            })
15698            .await;
15699        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15700        assert_eq!(Instant::now(), tick);
15701        tokio::time::advance(Duration::from_millis(1)).await;
15702        let mut started = false;
15703        wait_for("successful HTTP probe", || {
15704            started |= started_rx.try_recv().is_ok();
15705            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15706                .then_some(())
15707        })
15708        .await;
15709        assert!(started, "added HTTP check must start at its first deadline");
15710        assert_eq!(
15711            actor.module.status().unwrap().health.status,
15712            SupervisorHealthStatus::Ok
15713        );
15714        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15715        actor.health.http = None;
15716        let turns = actor.turns();
15717        actor
15718            .module
15719            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15720            .await
15721            .unwrap();
15722        let parked = actor
15723            .wait_for_select(|checkpoint| {
15724                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15725            })
15726            .await;
15727        assert_eq!(parked.wake_after, None);
15728        let turns = actor.turns();
15729        tokio::time::advance(Duration::from_secs(60)).await;
15730        assert_eq!(
15731            actor.turns(),
15732            turns,
15733            "removed HTTP check must park the actor"
15734        );
15735        assert!(
15736            started_rx.try_recv().is_err(),
15737            "removed HTTP check must stay disarmed"
15738        );
15739        assert_eq!(
15740            actor.module.status().unwrap().health,
15741            ModuleHealthStatus::default()
15742        );
15743        peer.abort();
15744    }
15745}
15746
15747#[cfg(test)]
15748mod drain_busy_reprobe_tests {
15749    //! The drain's busy-gauge wait, driven against an in-process module
15750    //! connection whose health answers the test scripts. Time is paused, so
15751    //! every elapsed duration below is exact virtual time.
15752    use super::*;
15753    use std::collections::BTreeMap;
15754    use std::sync::Mutex as StdMutex;
15755    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15756
15757    const MODULE_ID: &str = "drain-reprobe-module";
15758    const CONNECTION: u64 = 7;
15759    const GAUGE: &str = "runs_in_flight";
15760    /// The production default for both the health cadence and the drain
15761    /// ceiling, which is the pairing that used to cost a full ceiling.
15762    const CADENCE: Duration = Duration::from_secs(30);
15763    const CEILING: Duration = Duration::from_secs(30);
15764
15765    /// How the scripted module answers each `health.check`.
15766    #[derive(Clone, Copy)]
15767    enum Answer {
15768        /// Report the gauge busy until this instant, quiet from then on, after
15769        /// taking `latency` to answer.
15770        BusyUntil { until: Instant, latency: Duration },
15771        /// Answer at once, with metrics that leave the declared gauge out.
15772        OmitsGauge,
15773        /// Never answer.
15774        Silent,
15775    }
15776
15777    struct Drain {
15778        forwarding: Arc<ForwardingTable>,
15779        runtime: SupervisorRuntimeConfig,
15780        endpoint: crate::ModuleEndpointId,
15781        /// When each probe reached the module.
15782        probes: Arc<StdMutex<Vec<Instant>>>,
15783        responder: tokio::task::JoinHandle<()>,
15784    }
15785
15786    impl Drain {
15787        fn start(answer: Answer) -> Self {
15788            let registry = Arc::new(Registry::default());
15789            let forwarding = Arc::new(ForwardingTable::default());
15790            let health = HealthConfig {
15791                cadence: CADENCE,
15792                ..HealthConfig::default()
15793            };
15794            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15795                .with_forwarding(forwarding.clone())
15796                .with_health_config(health)
15797                .runtime_config();
15798            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15799            let endpoint = forwarding
15800                .register_module_connection(
15801                    ConnectionId::new(CONNECTION),
15802                    MODULE_ID.to_string(),
15803                    subc_protocol::PROTOCOL_VERSION,
15804                    Concurrency::ModuleManaged,
15805                    FrameSink::new(tx),
15806                )
15807                .unwrap();
15808            let probes = Arc::new(StdMutex::new(Vec::new()));
15809            let responder = tokio::spawn({
15810                let forwarding = forwarding.clone();
15811                let probes = probes.clone();
15812                async move {
15813                    while let Some(frame) = rx.recv().await {
15814                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15815                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15816                        probes.lock().unwrap().push(Instant::now());
15817                        let metrics = match answer {
15818                            Answer::BusyUntil { until, latency } => {
15819                                sleep(latency).await;
15820                                let busy = u64::from(Instant::now() < until);
15821                                serde_json::json!({ GAUGE: busy })
15822                            }
15823                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15824                            Answer::Silent => continue,
15825                        };
15826                        // A probe abandoned at its deadline has no RPC left to
15827                        // complete, so a late answer may be refused.
15828                        let _ = forwarding.complete_module_control_rpc(
15829                            ConnectionId::new(CONNECTION),
15830                            frame.header.corr,
15831                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15832                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15833                                status: HealthStatus::Ok,
15834                                detail: None,
15835                                metrics: Some(metrics),
15836                            }),
15837                        );
15838                    }
15839                }
15840            });
15841            Self {
15842                forwarding,
15843                runtime,
15844                endpoint,
15845                probes,
15846                responder,
15847            }
15848        }
15849
15850        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15851        /// how long it took.
15852        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15853            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15854            (drained, elapsed)
15855        }
15856
15857        /// [`Self::run`], also returning what the drain's probes read.
15858        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15859            let started = Instant::now();
15860            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15861            let drained = wait_for_forwarding_quiescence(
15862                &self.forwarding,
15863                MODULE_ID,
15864                &self.runtime,
15865                self.endpoint,
15866                started + ceiling,
15867                &[GAUGE.to_string()],
15868                DrainScope::Active,
15869                &mut trace,
15870            )
15871            .await
15872            .unwrap();
15873            (drained, started.elapsed(), trace)
15874        }
15875
15876        fn probes(&self) -> Vec<Instant> {
15877            self.probes.lock().unwrap().clone()
15878        }
15879    }
15880
15881    impl Drop for Drain {
15882        fn drop(&mut self) {
15883            self.responder.abort();
15884        }
15885    }
15886
15887    #[tokio::test(start_paused = true)]
15888    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15889        let drain = Drain::start(Answer::BusyUntil {
15890            until: Instant::now() + Duration::from_secs(1),
15891            latency: Duration::ZERO,
15892        });
15893        let (drained, elapsed) = drain.run(CEILING).await;
15894        assert!(
15895            drained,
15896            "gauges that read zero during the drain must let it finish drained, \
15897             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15898        );
15899        assert!(
15900            elapsed < Duration::from_secs(3),
15901            "drain took {elapsed:?} after the gauges went quiet at 1s"
15902        );
15903        let probes = drain.probes();
15904        assert!(
15905            probes.len() >= 2,
15906            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15907        );
15908    }
15909
15910    /// The same shape in real time on a multi-threaded runtime: the first
15911    /// probe is answered busy at once, within a millisecond of the drain
15912    /// starting, and later probes read idle. The drain must keep probing
15913    /// after that first instant answer and end soon after the gauge falls.
15914    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15915    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15916        let drain = Drain::start(Answer::BusyUntil {
15917            until: Instant::now() + Duration::from_millis(600),
15918            latency: Duration::ZERO,
15919        });
15920        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15921        assert!(drained, "the gauge fell at 600ms: {trace}");
15922        assert!(
15923            elapsed < Duration::from_secs(3),
15924            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15925        );
15926        let probes = drain.probes();
15927        assert!(
15928            probes.len() >= 2 && trace.probes as usize == probes.len(),
15929            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15930        );
15931    }
15932
15933    #[tokio::test(start_paused = true)]
15934    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15935        let started = Instant::now();
15936        let drain = Drain::start(Answer::BusyUntil {
15937            until: started + Duration::from_secs(3600),
15938            latency: Duration::ZERO,
15939        });
15940        let (drained, elapsed) = drain.run(CEILING).await;
15941        assert!(!drained, "a module busy throughout must not drain");
15942        assert!(
15943            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15944            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15945        );
15946        // Re-probing is paced by the interval, not a busy loop: at most one
15947        // probe per interval across the ceiling, plus the first at time zero.
15948        let probes = drain.probes();
15949        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15950        assert!(
15951            probes.len() as u128 <= max_probes,
15952            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15953            probes.len()
15954        );
15955        for pair in probes.windows(2) {
15956            assert!(
15957                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15958                "consecutive probes {:?} apart",
15959                pair[1] - pair[0]
15960            );
15961        }
15962    }
15963
15964    #[tokio::test(start_paused = true)]
15965    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15966        // Each answer takes longer than the re-probe interval; the next probe
15967        // must wait for the previous answer and then the interval.
15968        let latency = Duration::from_secs(1);
15969        let drain = Drain::start(Answer::BusyUntil {
15970            until: Instant::now() + Duration::from_secs(3600),
15971            latency,
15972        });
15973        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15974        assert!(!drained);
15975        let probes = drain.probes();
15976        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15977        for pair in probes.windows(2) {
15978            assert!(
15979                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15980                "a probe started {:?} after the previous one, before it could finish",
15981                pair[1] - pair[0]
15982            );
15983        }
15984    }
15985
15986    #[tokio::test(start_paused = true)]
15987    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15988        let drain = Drain::start(Answer::Silent);
15989        let ceiling = Duration::from_secs(2);
15990        let (drained, elapsed) = drain.run(ceiling).await;
15991        assert!(!drained, "an unanswered probe counts as busy");
15992        assert!(
15993            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15994            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15995        );
15996        assert_eq!(
15997            drain.probes().len(),
15998            1,
15999            "the unanswered probe holds the only slot until the ceiling"
16000        );
16001    }
16002
16003    /// Every event logged while it is installed, as `message` plus fields.
16004    #[derive(Clone, Default)]
16005    struct LogCapture {
16006        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16007    }
16008
16009    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16010        fn on_event(
16011            &self,
16012            event: &tracing::Event<'_>,
16013            _context: tracing_subscriber::layer::Context<'_, S>,
16014        ) {
16015            struct Fields(BTreeMap<String, String>);
16016            impl tracing::field::Visit for Fields {
16017                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16018                    self.0
16019                        .insert(field.name().to_string(), format!("{value:?}"));
16020                }
16021            }
16022            let mut fields = Fields(BTreeMap::new());
16023            event.record(&mut fields);
16024            self.events.lock().unwrap().push(fields.0);
16025        }
16026    }
16027
16028    impl LogCapture {
16029        fn install(&self) -> tracing::subscriber::DefaultGuard {
16030            use tracing_subscriber::layer::SubscriberExt;
16031            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16032        }
16033
16034        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16035            self.events
16036                .lock()
16037                .unwrap()
16038                .iter()
16039                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16040                .cloned()
16041                .collect()
16042        }
16043    }
16044
16045    /// A module that declares a busy gauge but leaves it out of its health
16046    /// answers holds a drain to its ceiling. The timeout line must say so by
16047    /// name, and the drain must warn about the missing gauge once, not on
16048    /// every probe.
16049    #[tokio::test(start_paused = true)]
16050    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16051        let capture = LogCapture::default();
16052        let _guard = capture.install();
16053        let drain = Drain::start(Answer::OmitsGauge);
16054        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16055        assert!(!drained, "a gauge never reported cannot read quiet");
16056        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16057
16058        let missing = capture.with_message("lacks a declared busy gauge");
16059        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16060        assert_eq!(
16061            missing[0].get("missing_gauges").map(String::as_str),
16062            Some(GAUGE)
16063        );
16064
16065        warn_drain_timed_out(
16066            MODULE_ID,
16067            Duration::from_secs(2),
16068            RouteCloseReason::Restart,
16069            &Default::default(),
16070            &trace,
16071        );
16072        let timeouts = capture.with_message("route drain timed out");
16073        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16074        let gauges = &timeouts[0]["busy_gauges"];
16075        assert!(
16076            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16077            "the timeout line must name the gauge and why it read busy: {gauges}"
16078        );
16079        assert!(
16080            gauges.ends_with("; no value ever read"),
16081            "no value was ever read for the gauge: {gauges}"
16082        );
16083        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16084    }
16085
16086    /// The trace names a busy value and when it was read. For a probe that
16087    /// never got an answer, it names the probe's error kind and says that no
16088    /// value was ever read.
16089    #[tokio::test(start_paused = true)]
16090    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16091        let busy = Drain::start(Answer::BusyUntil {
16092            until: Instant::now() + Duration::from_secs(3600),
16093            latency: Duration::ZERO,
16094        });
16095        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16096        let rendered = trace.to_string();
16097        assert!(
16098            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16099                && rendered.contains("; last value non_zero(1) @"),
16100            "{rendered}"
16101        );
16102
16103        let silent = Drain::start(Answer::Silent);
16104        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16105        // The probe is capped at the 1 s ceiling, and its reading is stamped
16106        // when it gave up.
16107        assert_eq!(
16108            trace.to_string(),
16109            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16110        );
16111        assert_eq!(trace.probes, 1);
16112    }
16113
16114    /// A gauge value that is not a count reads busy and is shown as sent.
16115    #[test]
16116    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16117        let gauges = [GAUGE.to_string()];
16118        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16119        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16120        assert_eq!(
16121            busy_gauge_observation(&verdicts),
16122            BusyGaugeObservation::Busy
16123        );
16124        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16125        assert_eq!(
16126            busy_gauge_observation(&zero),
16127            BusyGaugeObservation::Quiescent
16128        );
16129        let none = gauge_verdicts(None, &gauges);
16130        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16131    }
16132}
16133
16134#[cfg(test)]
16135mod health_evidence_tests {
16136    use super::{HealthProbeError, HealthProbeEvidence};
16137    use std::collections::HashSet;
16138
16139    /// The evidential asymmetry, asserted rather than described.
16140    ///
16141    /// Exactly ONE observation is proof a module cannot serve, and the one that
16142    /// fires under CPU starvation is not it. Before the split, all fifteen
16143    /// construction sites collapsed into a single String, so a timeout carried the
16144    /// same weight as a dead lane -- which is how a healthy module was restarted
16145    /// three times in one day.
16146    #[test]
16147    fn only_a_dead_lane_is_proof_of_death() {
16148        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16149        // Three non-proof classes, each for a different reason: silence is
16150        // consistent with health, a bad answer proves the module ALIVE, and a
16151        // daemon-side fault never reached the module at all.
16152        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16153        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16154        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16155    }
16156
16157    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
16158    ///
16159    /// A shared label renders two different observations identically in the line an
16160    /// operator reads after an unexplained restart -- the exact confusion this
16161    /// change removes.
16162    #[test]
16163    fn every_evidence_class_has_a_distinct_label() {
16164        let labels = [
16165            HealthProbeError::lane_dead("").label(),
16166            HealthProbeError::no_answer("").label(),
16167            HealthProbeError::bad_answer("").label(),
16168            HealthProbeError::misconfigured("").label(),
16169        ];
16170        let unique: HashSet<_> = labels.iter().collect();
16171        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16172    }
16173
16174    /// The class is additional information, not a replacement.
16175    ///
16176    /// An operator needs both "this was silence" and the specific text saying how
16177    /// long we waited; a classification that swallowed the message would trade one
16178    /// missing distinction for another.
16179    #[test]
16180    fn classification_preserves_the_original_message() {
16181        let err = HealthProbeError::no_answer("module did not answer within 5s");
16182        assert_eq!(err.to_string(), "module did not answer within 5s");
16183        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16184    }
16185}
16186
16187#[cfg(test)]
16188mod health_tombstone_tests {
16189    use std::{path::PathBuf, sync::Arc, time::Duration};
16190
16191    use subc_protocol::{
16192        manifest::Concurrency,
16193        session::{HealthStatus, ModuleControlResponse},
16194    };
16195    use tokio::sync::mpsc;
16196
16197    use super::{
16198        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16199        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16200    };
16201    use crate::{
16202        control::ControlHandler,
16203        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16204        registry::{ConnectionId, Registry},
16205        router::FrameSink,
16206    };
16207
16208    struct ProbeHarness {
16209        spec: ModuleSpec,
16210        runtime: SupervisorRuntimeConfig,
16211        forwarding: Arc<ForwardingTable>,
16212        module_connection: ConnectionId,
16213        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16214        handler: ControlHandler,
16215        module: super::SupervisedModule,
16216    }
16217
16218    fn probe_harness() -> ProbeHarness {
16219        let registry = Arc::new(Registry::default());
16220        let forwarding = Arc::new(ForwardingTable::default());
16221        let supervisor_handle = super::SupervisorHandle::new();
16222        let health = HealthConfig {
16223            http: None,
16224            cadence: Duration::from_secs(30),
16225            deadline: Duration::from_secs(5),
16226            failure_threshold: 3,
16227            on_degraded: HealthAction::Report,
16228            on_failing: HealthAction::Report,
16229            critical: false,
16230        };
16231        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16232            .with_forwarding(Arc::clone(&forwarding))
16233            .with_handle(supervisor_handle.clone())
16234            .with_health_config(health);
16235        let spec = ModuleSpec {
16236            module_id: "late-health-module".to_string(),
16237            program: PathBuf::from("disabled-module"),
16238            args: Vec::new(),
16239            env: Vec::new(),
16240            reserved: false,
16241            reserved_prefixes: Vec::new(),
16242            protocol: ModuleProtocol::Subc,
16243            overlap: Default::default(),
16244            carry_containment: false,
16245        };
16246        let module = supervisor
16247            .supervise_configured(spec.clone(), false)
16248            .unwrap();
16249        let runtime = supervisor.runtime_config();
16250        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16251            .with_supervisor(supervisor_handle);
16252        let module_connection = ConnectionId::new(700);
16253        let (module_tx, module_rx) = mpsc::channel(8);
16254        forwarding
16255            .register_module_connection(
16256                module_connection,
16257                spec.module_id.clone(),
16258                subc_protocol::PROTOCOL_VERSION,
16259                Concurrency::ModuleManaged,
16260                FrameSink::new(module_tx),
16261            )
16262            .unwrap();
16263
16264        ProbeHarness {
16265            spec,
16266            runtime,
16267            forwarding,
16268            module_connection,
16269            module_rx,
16270            handler,
16271            module,
16272        }
16273    }
16274
16275    async fn finish_after(
16276        harness: &mut ProbeHarness,
16277        stall: Duration,
16278    ) -> ModuleControlRpcCompletion {
16279        assert!(stall > harness.runtime.health.deadline);
16280        let deadline = harness.runtime.health.deadline;
16281        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16282        let answer = async {
16283            let frame = harness.module_rx.recv().await.expect("health.check frame");
16284            tokio::time::advance(deadline).await;
16285            tokio::task::yield_now().await;
16286            tokio::time::advance(stall - deadline).await;
16287            harness
16288                .forwarding
16289                .complete_module_control_rpc(
16290                    harness.module_connection,
16291                    frame.header.corr,
16292                    Some("health.check"),
16293                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16294                        status: HealthStatus::Ok,
16295                        detail: None,
16296                        metrics: None,
16297                    }),
16298                )
16299                .unwrap()
16300        };
16301        let (probe_result, completion) = tokio::join!(probe, answer);
16302        let err = probe_result.expect_err("probe must miss its deadline");
16303        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16304        completion
16305    }
16306
16307    async fn time_out_without_answer(harness: &mut ProbeHarness) {
16308        let deadline = harness.runtime.health.deadline;
16309        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16310        let exhaust_deadline = async {
16311            let _frame = harness.module_rx.recv().await.expect("health.check frame");
16312            tokio::time::advance(deadline).await;
16313            tokio::task::yield_now().await;
16314        };
16315        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16316        let err = probe_result.expect_err("probe must miss its deadline");
16317        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16318    }
16319
16320    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16321        let registry = Arc::clone(&harness.module.inner.registry);
16322        let snapshot = Arc::clone(&harness.module.inner.snapshot);
16323        let process_liveness = super::SupervisorProcessLiveness::default();
16324        let mut child = None;
16325        let cycle = super::run_health_probe_cycle(
16326            &harness.spec,
16327            &harness.runtime,
16328            &registry,
16329            &process_liveness,
16330            &snapshot,
16331            &mut child,
16332        );
16333        let peer = async {
16334            let frame = harness.module_rx.recv().await.expect("health.check frame");
16335            if answer {
16336                harness
16337                    .forwarding
16338                    .complete_module_control_rpc(
16339                        harness.module_connection,
16340                        frame.header.corr,
16341                        Some("health.check"),
16342                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16343                            status: HealthStatus::Ok,
16344                            detail: None,
16345                            metrics: Some(serde_json::json!({"ready": true})),
16346                        }),
16347                    )
16348                    .unwrap();
16349            } else {
16350                tokio::time::advance(harness.runtime.health.deadline).await;
16351                tokio::task::yield_now().await;
16352            }
16353        };
16354        tokio::join!(cycle, peer);
16355    }
16356
16357    #[tokio::test(start_paused = true)]
16358    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16359        let mut harness = probe_harness();
16360        // Drive the probe cycle directly with an in-memory wire peer. Stop the
16361        // disabled module's monitor so only this test owns lifecycle transitions;
16362        // no OS process is launched, and a restart is observed at scheduling.
16363        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16364        monitor.abort();
16365        let _ = monitor.await;
16366        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16367            state.enabled = true;
16368            state.state = super::ModuleState::Running;
16369            state.process_alive = true;
16370        })
16371        .unwrap();
16372
16373        run_probe_cycle(&mut harness, true).await;
16374        assert_eq!(
16375            harness.module.status().unwrap().health.status,
16376            super::SupervisorHealthStatus::Ok
16377        );
16378
16379        for failures in 1..harness.runtime.health.failure_threshold {
16380            run_probe_cycle(&mut harness, false).await;
16381            let status = harness.module.status().unwrap();
16382            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16383            assert_eq!(status.health.consecutive_failures, failures);
16384            assert!(status.health.last_probe_ms.is_some());
16385            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16386            assert!(status.health.metrics.is_none());
16387            assert_eq!(status.state, super::ModuleState::Running);
16388            assert!(status.process_alive);
16389            assert_eq!(status.restart_count, 0);
16390            assert_eq!(status.lifetime_restarts, 0);
16391            assert!(status.health.last_action.is_none());
16392            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16393        }
16394
16395        run_probe_cycle(&mut harness, true).await;
16396        let recovered = harness.module.status().unwrap();
16397        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16398        assert_eq!(recovered.health.consecutive_failures, 0);
16399        assert!(recovered.health.detail.is_none());
16400        assert_eq!(
16401            recovered.health.metrics,
16402            Some(serde_json::json!({"ready": true}))
16403        );
16404        assert_eq!(recovered.lifetime_restarts, 0);
16405
16406        for failures in 1..=harness.runtime.health.failure_threshold {
16407            run_probe_cycle(&mut harness, false).await;
16408            let status = harness.module.status().unwrap();
16409            assert_eq!(status.health.consecutive_failures, failures);
16410            if failures < harness.runtime.health.failure_threshold {
16411                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16412                assert_eq!(status.state, super::ModuleState::Running);
16413                assert_eq!(status.lifetime_restarts, 0);
16414                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16415            } else {
16416                assert_eq!(
16417                    status.health.status,
16418                    super::SupervisorHealthStatus::Unresponsive
16419                );
16420                assert_eq!(status.state, super::ModuleState::Restarting);
16421                assert_eq!(status.restart_count, 1);
16422                assert_eq!(status.lifetime_restarts, 1);
16423                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16424                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16425            }
16426        }
16427    }
16428
16429    #[tokio::test(start_paused = true)]
16430    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16431        let mut harness = probe_harness();
16432
16433        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16434        let first_latency = match &first {
16435            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16436            other => panic!("late answer was not retained: {other:?}"),
16437        };
16438        assert!(harness.handler.observe_module_control_completion(first));
16439
16440        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16441        let second_latency = match &second {
16442            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16443            other => panic!("late answer was not retained: {other:?}"),
16444        };
16445        assert!(harness.handler.observe_module_control_completion(second));
16446
16447        assert_eq!(first_latency, Duration::from_secs(8));
16448        assert_eq!(
16449            second_latency - first_latency,
16450            Duration::from_secs(3),
16451            "latency must grow linearly with the additional stall"
16452        );
16453        let health = harness.module.status().unwrap().health;
16454        assert_eq!(health.late_answer_count, 2);
16455        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16456    }
16457
16458    /// A module that answers every probe late must never march to the kill
16459    /// threshold: the late answer proves it is alive, so it must clear the miss
16460    /// streak the timeout recorded. Without the reset, a CPU-starved module
16461    /// that serves every probe seconds past the deadline accumulates
16462    /// `consecutive_failures` to the threshold and is killed — the exact
16463    /// sequence from the 2026-08-14 aft disable, where the daemon logged
16464    /// "proves the module is alive" five times while counting five misses.
16465    #[tokio::test(start_paused = true)]
16466    async fn late_answer_clears_the_consecutive_failure_streak() {
16467        let mut harness = probe_harness();
16468
16469        // Timeout recorded first: the probe path saw no answer in time.
16470        time_out_without_answer(&mut harness).await;
16471        harness
16472            .module
16473            .record_health_probe_failure_for_test("[no-answer] test miss")
16474            .unwrap();
16475        assert_eq!(
16476            harness.module.status().unwrap().health.consecutive_failures,
16477            1,
16478            "precondition: the miss must be on the streak before the late answer"
16479        );
16480
16481        // The stalled reply then lands: proof of life.
16482        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16483        assert!(matches!(
16484            late,
16485            ModuleControlRpcCompletion::LateHealthAnswer { .. }
16486        ));
16487        assert!(harness.handler.observe_module_control_completion(late));
16488
16489        let health = harness.module.status().unwrap().health;
16490        assert_eq!(
16491            health.consecutive_failures, 0,
16492            "a late answer is an answer: the streak must reset"
16493        );
16494        assert_eq!(health.late_answer_count, 1);
16495    }
16496
16497    #[tokio::test(start_paused = true)]
16498    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16499        let mut harness = probe_harness();
16500
16501        for _ in 0..20 {
16502            time_out_without_answer(&mut harness).await;
16503            assert_eq!(
16504                harness.forwarding.health_probe_tombstone_count().unwrap(),
16505                1
16506            );
16507        }
16508    }
16509}
16510
16511#[cfg(test)]
16512mod child_env_tests {
16513    #[cfg(windows)]
16514    use super::{
16515        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16516    };
16517    #[cfg(windows)]
16518    use std::sync::{Arc, OnceLock};
16519    #[cfg(windows)]
16520    #[test]
16521    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16522        let spec = spec(Vec::new());
16523        let mut command = Command::new("unused");
16524        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16525            .unwrap()
16526            .unwrap();
16527        let name = command
16528            .as_std()
16529            .get_envs()
16530            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16531            .and_then(|(_, value)| value)
16532            .unwrap();
16533        assert_eq!(name, OsStr::new(handoff.name()));
16534        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16535        assert!(command.as_std().get_envs().any(|(key, value)| key
16536            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16537            && value == Some(OsStr::new("env"))));
16538        assert!(command
16539            .as_std()
16540            .get_envs()
16541            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16542    }
16543
16544    #[cfg(windows)]
16545    #[test]
16546    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16547        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16548        snapshot.process_alive = true;
16549        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16550        assert_eq!(
16551            observed_launch_nonce_source(&snapshot).as_deref(),
16552            Some("env")
16553        );
16554        let consumed = Arc::new(OnceLock::new());
16555        snapshot.pipe_consumed = Some(consumed.clone());
16556        assert_eq!(
16557            observed_launch_nonce_source(&snapshot).as_deref(),
16558            Some("env")
16559        );
16560        consumed.set(()).unwrap();
16561        assert_eq!(
16562            observed_launch_nonce_source(&snapshot).as_deref(),
16563            Some("pipe")
16564        );
16565        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16566        assert_eq!(
16567            observed_launch_nonce_source(&snapshot).as_deref(),
16568            Some("env")
16569        );
16570        clear_current_process_facts(&mut snapshot);
16571        assert_eq!(observed_launch_nonce_source(&snapshot), None);
16572    }
16573    use super::{
16574        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16575        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16576        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16577    };
16578    use std::{ffi::OsStr, path::PathBuf};
16579    use tokio::process::Command;
16580
16581    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16582        ModuleSpec {
16583            module_id: "env-plan".to_string(),
16584            program: PathBuf::from("/nonexistent"),
16585            args: Vec::new(),
16586            env,
16587            reserved: false,
16588            reserved_prefixes: Vec::new(),
16589            protocol: ModuleProtocol::Subc,
16590            overlap: Default::default(),
16591            carry_containment: false,
16592        }
16593    }
16594
16595    #[test]
16596    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16597        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16598        let digest = LaunchSpecDigest::from_spec(&base);
16599        let mut reordered = base.clone();
16600        reordered.env.reverse();
16601        reordered.module_id = "different-id".into();
16602        reordered.reserved = true;
16603        reordered.overlap = Default::default();
16604        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16605        let mut delimiter_collision = base.clone();
16606        delimiter_collision.env[0] = ("A=B".into(), "C".into());
16607        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16608        let mut changed = base.clone();
16609        changed.protocol = ModuleProtocol::None;
16610        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16611        changed = base.clone();
16612        changed.args.push("--flag".into());
16613        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16614        changed = base.clone();
16615        changed.program = PathBuf::from("/other");
16616        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16617        changed = base.clone();
16618        changed.env[1].1 = "other".into();
16619        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16620        assert!(!format!("{digest:?}").contains("B=C"));
16621        assert!(!format!("{digest:?}").contains("[u8"));
16622    }
16623
16624    #[cfg(unix)]
16625    #[test]
16626    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16627        use std::os::unix::ffi::OsStringExt;
16628        let mut left = spec(Vec::new());
16629        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16630        let mut right = left.clone();
16631        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16632        assert_eq!(
16633            left.program.to_string_lossy(),
16634            right.program.to_string_lossy()
16635        );
16636        assert_ne!(
16637            LaunchSpecDigest::from_spec(&left),
16638            LaunchSpecDigest::from_spec(&right)
16639        );
16640    }
16641
16642    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
16643    /// one still gets its own.
16644    ///
16645    /// This is the narrow goal `env_clear()` was reached for, and the reason the
16646    /// fix is `env_remove` rather than deleting the line: an operator's ambient
16647    /// filter silently becoming an unconfigured module's log level is a real
16648    /// defect, just a much smaller one than clearing the environment.
16649    ///
16650    /// Asserted on the command plan rather than a spawned child because proving
16651    /// the ABSENCE of an inherited variable needs the parent's environment
16652    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
16653    /// removal as `(key, None)`, which is exactly the distinction wanted: not
16654    /// "absent because nobody set it" but "explicitly unset for the child".
16655    #[test]
16656    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16657        let mut command = Command::new("/nonexistent");
16658        apply_child_env(&mut command, &spec(Vec::new()));
16659        let removed = command
16660            .as_std()
16661            .get_envs()
16662            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16663        assert!(
16664            removed,
16665            "ambient CK_LOG must be explicitly removed for an unconfigured module"
16666        );
16667
16668        let mut configured = Command::new("/nonexistent");
16669        apply_child_env(
16670            &mut configured,
16671            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16672        );
16673        let effective = configured
16674            .as_std()
16675            .get_envs()
16676            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16677            .last()
16678            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16679        assert_eq!(
16680            effective,
16681            Some(Some("debug".to_string())),
16682            "a module's configured CK_LOG must survive the ambient removal"
16683        );
16684    }
16685
16686    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
16687    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
16688    /// the same reason as the CK_LOG test above.
16689    ///
16690    /// The argument is the load-bearing half: a stock binary exits on an
16691    /// unknown flag before it listens, so with `--subc` appended the mode
16692    /// could not supervise the one process it exists for. Found by the first
16693    /// conformance run (nats-server: `flag provided but not defined: -subc`).
16694    #[test]
16695    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16696        let connection_file = std::path::Path::new("/run/subc-connection.json");
16697        let handle = SupervisorHandle::new();
16698
16699        let mut none_spec = spec(Vec::new());
16700        none_spec.protocol = ModuleProtocol::None;
16701        let mut none = Command::new("/nonexistent");
16702        let none_handoff =
16703            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16704                .expect("protocol-none spawn args apply");
16705        assert!(
16706            none_handoff.is_none(),
16707            "protocol:none spawn must not receive a nonce descriptor"
16708        );
16709        #[cfg(windows)]
16710        assert!(!none.as_std().get_envs().any(|(key, value)| key
16711            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16712            && value.is_some()));
16713        assert!(
16714            !none.as_std().get_envs().any(|(key, value)| key
16715                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16716                && value.is_some()),
16717            "protocol:none spawn must not name a nonce descriptor"
16718        );
16719        let none_args: Vec<String> = none
16720            .as_std()
16721            .get_args()
16722            .map(|a| a.to_string_lossy().into_owned())
16723            .collect();
16724        assert!(
16725            !none_args.iter().any(|a| a == SUBC_ARG),
16726            "protocol:none argv must not carry --subc; got {none_args:?}"
16727        );
16728        let none_has_nonce = none
16729            .as_std()
16730            .get_envs()
16731            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16732        assert!(
16733            !none_has_nonce,
16734            "protocol:none spawn must not receive a launch nonce"
16735        );
16736        let none_has_module_id = none
16737            .as_std()
16738            .get_envs()
16739            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16740        assert!(
16741            none_has_module_id,
16742            "SUBC_MODULE_ID is inert and stays on every path"
16743        );
16744        assert!(
16745            handle.spawn_nonce(&none_spec.module_id).is_none(),
16746            "no nonce record for a process that will never present one"
16747        );
16748
16749        // Control: the subc-wire path is unchanged by the branch above.
16750        let wire_spec = spec(Vec::new());
16751        let mut wire = Command::new("/nonexistent");
16752        let wire_handoff =
16753            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16754                .expect("subc-wire spawn args apply");
16755        let wire_fd_env = wire
16756            .as_std()
16757            .get_envs()
16758            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16759            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16760        #[cfg(unix)]
16761        assert_eq!(
16762            wire_fd_env,
16763            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16764            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16765        );
16766        #[cfg(windows)]
16767        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16768        #[cfg(not(any(unix, windows)))]
16769        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16770        let wire_args: Vec<String> = wire
16771            .as_std()
16772            .get_args()
16773            .map(|a| a.to_string_lossy().into_owned())
16774            .collect();
16775        assert_eq!(
16776            wire_args,
16777            vec![
16778                SUBC_ARG.to_string(),
16779                connection_file.to_string_lossy().into_owned()
16780            ],
16781            "a subc-wire spawn still carries --subc <path>"
16782        );
16783        assert_eq!(
16784            wire.as_std()
16785                .get_envs()
16786                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16787            !cfg!(unix),
16788            "only Windows supplies the environment nonce"
16789        );
16790        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16791    }
16792
16793    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16794    /// spec tries to set it; only a swap candidate carries it.
16795    ///
16796    /// "Set it only on candidates" is not enough, because spawn applies the
16797    /// spec's env verbatim and the daemon's own environment is inherited: either
16798    /// could hand a plain restart the swap role, and a module reading it would
16799    /// warm on its long swap budget while callers wait. Asserted as an explicit
16800    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16801    /// test above gives.
16802    #[test]
16803    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16804        let role = |command: &Command| {
16805            command
16806                .as_std()
16807                .get_envs()
16808                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16809                .last()
16810                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16811        };
16812        let forged = spec(vec![(
16813            SUBC_SPAWN_ROLE_ENV.to_string(),
16814            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16815        )]);
16816
16817        let mut plain = Command::new("/nonexistent");
16818        apply_child_env(&mut plain, &forged);
16819        apply_spawn_role(&mut plain, SpawnRole::Plain);
16820        assert_eq!(
16821            role(&plain),
16822            Some(None),
16823            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16824        );
16825
16826        let mut candidate = Command::new("/nonexistent");
16827        apply_child_env(&mut candidate, &spec(Vec::new()));
16828        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16829        assert_eq!(
16830            role(&candidate),
16831            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16832        );
16833    }
16834
16835    /// Daemon-private capture retention keys never reach the child.
16836    ///
16837    /// cortexkit-log exposes retention as a Rust struct with no environment
16838    /// names, so these entries are supervisor metadata. Passing them through
16839    /// would invent a public child-process contract by accident.
16840    #[test]
16841    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16842        let mut command = Command::new("/nonexistent");
16843        apply_child_env(
16844            &mut command,
16845            &spec(vec![
16846                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16847                ("KEPT".to_string(), "yes".to_string()),
16848            ]),
16849        );
16850        let keys: Vec<String> = command
16851            .as_std()
16852            .get_envs()
16853            .filter(|(_, value)| value.is_some())
16854            .map(|(key, _)| key.to_string_lossy().into_owned())
16855            .collect();
16856        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16857        assert!(
16858            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16859            "daemon-private capture key leaked to the child: {keys:?}"
16860        );
16861    }
16862}
16863
16864#[cfg(test)]
16865mod jitter_tests {
16866    use super::jittered_health_delay;
16867    use std::{collections::HashSet, time::Duration};
16868
16869    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16870    /// that actually occur rather than invented ones.
16871    ///
16872    /// This is a SAMPLE, not a registry: the property under test is that distinct
16873    /// ids disperse, which holds for any set of distinct strings. Several entries
16874    /// are already historical (modules get renamed), and that costs nothing here --
16875    /// but it means a reader must not mistake this for the live module set, and a
16876    /// rename sweep will match it without there being anything to change.
16877    const FLEET: [&str; 14] = [
16878        "aft",
16879        "alfonso-core",
16880        "magic-context",
16881        "broca",
16882        "thalamus",
16883        "quota",
16884        "engram",
16885        "plexus",
16886        "cerebellum",
16887        "astrocyte",
16888        "synapse",
16889        "subc-mcp",
16890        "cortexkit-credentials",
16891        "subc-federation",
16892    ];
16893
16894    /// Probes must not converge after a fleet-wide restart.
16895    ///
16896    /// This is the property the jitter exists for: every module reconnects at
16897    /// once, and without dispersal all fourteen would then probe on the same
16898    /// tick forever. Nothing failed visibly when this went untested -- a
16899    /// convergent fleet still probes correctly, just in a burst, so the symptom
16900    /// is a periodic load spike that looks like whatever else is running.
16901    #[test]
16902    fn probe_delays_disperse_across_the_fleet() {
16903        let cadence = Duration::from_secs(30);
16904        let delays: HashSet<Duration> = FLEET
16905            .iter()
16906            .map(|id| jittered_health_delay(id, 0, cadence))
16907            .collect();
16908        assert_eq!(
16909            delays.len(),
16910            FLEET.len(),
16911            "every supervised module must land on its own probe offset"
16912        );
16913    }
16914
16915    /// The offset may only ever DELAY a probe, never bring it forward.
16916    ///
16917    /// A delay below the cadence would probe a module more often than
16918    /// configured, which is the opposite of what an operator asked for and
16919    /// would tighten the failure budget without anyone changing it.
16920    #[test]
16921    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16922        let cadence = Duration::from_secs(30);
16923        let span = cadence / 10;
16924        for id in FLEET {
16925            for probe_index in 0..8 {
16926                let delay = jittered_health_delay(id, probe_index, cadence);
16927                assert!(
16928                    delay >= cadence,
16929                    "{id}#{probe_index}: jitter must not shorten the cadence"
16930                );
16931                assert!(
16932                    delay < cadence + span,
16933                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16934                );
16935            }
16936        }
16937    }
16938
16939    /// A module keeps its offset across daemon restarts.
16940    ///
16941    /// The delay is derived rather than randomised precisely so a restart does
16942    /// not re-roll every module into a fresh chance of collision. A random
16943    /// source would satisfy the dispersal test above and quietly lose this.
16944    #[test]
16945    fn a_module_offset_is_stable_across_restarts() {
16946        let cadence = Duration::from_secs(30);
16947        for id in FLEET {
16948            assert_eq!(
16949                jittered_health_delay(id, 0, cadence),
16950                jittered_health_delay(id, 0, cadence),
16951                "{id}: the same module and probe index must produce the same offset"
16952            );
16953        }
16954    }
16955
16956    /// A zero cadence disables probing rather than producing a busy loop.
16957    #[test]
16958    fn zero_cadence_yields_zero_delay() {
16959        assert_eq!(
16960            jittered_health_delay("aft", 0, Duration::ZERO),
16961            Duration::ZERO
16962        );
16963    }
16964}
16965
16966#[cfg(all(test, target_os = "linux"))]
16967mod cgroup_placement_tests {
16968    use super::{
16969        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16970        SuperviseError, SupervisedChild,
16971    };
16972    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16973    use cortexkit_test_support::ScratchDir;
16974    use std::{
16975        fs, io,
16976        path::{Path, PathBuf},
16977        sync::{Arc, Mutex},
16978    };
16979    use tokio::process::Command;
16980
16981    #[tokio::test]
16982    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16983        use super::*;
16984        let dir = ScratchDir::new("unique-spawn-cgroups");
16985        let root = PathBuf::from(format!(
16986            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16987            std::process::id(),
16988            unix_ms_now()
16989        ));
16990        if let Err(error) = fs::create_dir(&root) {
16991            assert!(
16992                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16993                "required cgroup test cannot execute: {error}"
16994            );
16995            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16996            return;
16997        }
16998        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16999        let group_count = || {
17000            fs::read_dir(root.join("subc-modules"))
17001                .unwrap()
17002                .map(|entry| entry.unwrap().file_type().unwrap())
17003                .filter(|kind| kind.is_dir())
17004                .count()
17005        };
17006        let supervisor =
17007            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17008                .with_cgroup_placement(Some(placement.clone()));
17009        let runtime = supervisor.runtime_config();
17010        let mut spec = ModuleSpec {
17011            module_id: "unique-spawn".into(),
17012            program: PathBuf::from("/bin/sleep"),
17013            args: vec!["60".into()],
17014            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17015                .into_iter()
17016                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17017                .collect(),
17018            reserved: false,
17019            reserved_prefixes: vec![],
17020            protocol: ModuleProtocol::None,
17021            overlap: Default::default(),
17022            carry_containment: false,
17023        };
17024        let spawn = |spec: &ModuleSpec| {
17025            spawn_child(
17026                spec,
17027                None,
17028                None,
17029                &runtime.stderr_ring,
17030                None,
17031                &runtime.child_roster,
17032                Some(&placement),
17033            )
17034            .unwrap()
17035        };
17036        let mut live = spawn(&spec);
17037        for _ in 0..3 {
17038            // A new process can enter the old slot while retirement is pending.
17039            let next = spawn(&spec);
17040            assert_ne!(live.module_id, next.module_id);
17041            live.start_kill().unwrap();
17042            live.wait().await.unwrap();
17043            live = next;
17044            assert!(
17045                live.child.try_wait().unwrap().is_none(),
17046                "retiring the old slot must not kill the replacement"
17047            );
17048            assert_eq!(
17049                group_count(),
17050                1,
17051                "only the live spawn's cgroup should remain"
17052            );
17053        }
17054        supervisor.begin_daemon_shutdown();
17055        let reap = tokio::spawn(async move {
17056            live.wait().await.unwrap();
17057        });
17058        supervisor
17059            .end_children_for_daemon_shutdown(false, std::future::pending())
17060            .await;
17061        reap.await.unwrap();
17062        assert_eq!(group_count(), 0);
17063        // A normal exit uses the same tree-cleanup path as a killed spawn.
17064        spec.program = PathBuf::from("/bin/true");
17065        spec.args.clear();
17066        let fresh_roster = ChildRoster::default();
17067        let mut short = spawn_child(
17068            &spec,
17069            None,
17070            None,
17071            &runtime.stderr_ring,
17072            None,
17073            &fresh_roster,
17074            Some(&placement),
17075        )
17076        .unwrap();
17077        short.wait().await.unwrap();
17078        assert_eq!(group_count(), 0);
17079        spec.module_id = "_".repeat(255);
17080        let mut long_id = spawn_child(
17081            &spec,
17082            None,
17083            None,
17084            &runtime.stderr_ring,
17085            None,
17086            &fresh_roster,
17087            Some(&placement),
17088        )
17089        .unwrap();
17090        long_id.wait().await.unwrap();
17091        assert_eq!(
17092            group_count(),
17093            0,
17094            "valid long module IDs must not exceed cgroup NAME_MAX"
17095        );
17096        fs::remove_dir(root.join("subc-modules")).unwrap();
17097        fs::remove_dir(root).unwrap();
17098        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17099    }
17100
17101    #[test]
17102    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17103        let path = Path::new("/definitely-missing-subc-cgroup");
17104        let mut command = Command::new("true");
17105        let error = apply_cgroup_placement(
17106            &mut command,
17107            &ModuleSpec {
17108                module_id: "broken-cgroup".to_string(),
17109                program: PathBuf::from("true"),
17110                args: Vec::new(),
17111                env: Vec::new(),
17112                reserved: false,
17113                reserved_prefixes: Vec::new(),
17114                protocol: ModuleProtocol::Subc,
17115                overlap: Default::default(),
17116                carry_containment: false,
17117            },
17118            path,
17119        )
17120        .expect_err("a parent cgroup open failure must reject the supervised spawn");
17121        let reason = error.to_string();
17122
17123        assert!(
17124            matches!(error, SuperviseError::Cgroup { .. }),
17125            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17126        );
17127        assert!(
17128            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17129            "parent cgroup open failure must name cgroup.procs: {reason}"
17130        );
17131    }
17132
17133    #[tokio::test]
17134    async fn reaping_a_child_removes_its_empty_module_cgroup() {
17135        let root = ScratchDir::new("supervisor-reap-cgroup");
17136        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17137        let placement = subc_cgroup::prepare_at(&root)
17138            .expect("prepare scratch cgroup root")
17139            .expect("scratch root has a cgroup.procs marker");
17140        let module_id = "reaped-module";
17141        let module = placement
17142            .module_path(module_id)
17143            .expect("create scratch module cgroup");
17144        let child = Command::new("true")
17145            .env("XDG_DATA_HOME", root.path())
17146            .env("XDG_RUNTIME_DIR", root.path())
17147            .env("XDG_CONFIG_HOME", root.path())
17148            .spawn()
17149            .expect("spawn short-lived child");
17150        let pid = child.id().expect("spawned child has pid");
17151        let mut child = SupervisedChild {
17152            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17153            child: crate::supervise::containment::ProcessChild::plain(child, pid),
17154            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17155            protocol: ModuleProtocol::Subc,
17156            module_id: module_id.to_string(),
17157            cgroup_placement: Some(placement),
17158            stdout_pump: None,
17159            stderr_pump: None,
17160            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17161            spawned_at: tokio::time::Instant::now(),
17162            registration_deadline: None,
17163            registration_watch: None,
17164            spawned_at_ms: 0,
17165            spawned_from: PathBuf::from("true"),
17166            launch_spec_digest: LaunchSpecDigest([0; 32]),
17167            spawned_file_identity: None,
17168            process_start_time: None,
17169            process_identity: None,
17170            pid,
17171            roster_guard: None,
17172            #[cfg(target_os = "macos")]
17173            privacy_exec: None,
17174            spawn_failure: None,
17175        };
17176
17177        child.wait().await.expect("reap short-lived child");
17178
17179        assert!(
17180            !module.exists(),
17181            "waiting for a non-carried child must remove its empty cgroup before returning"
17182        );
17183        child.finish_exit(false).await.unwrap();
17184    }
17185
17186    #[test]
17187    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17188        let root = ScratchDir::new("supervisor-non-empty-cgroup");
17189        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17190        let placement = subc_cgroup::prepare_at(&root)
17191            .expect("prepare scratch cgroup root")
17192            .expect("scratch root has a cgroup.procs marker");
17193        let module = placement
17194            .module_path("surviving-module")
17195            .expect("create scratch module cgroup");
17196        fs::write(module.join("surviving-process"), b"still present")
17197            .expect("make scratch cgroup non-empty");
17198        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17199
17200        remove_module_cgroup(&placement, "surviving-module");
17201
17202        let logs = crate::router::test_log::captured_logs(&logs);
17203        assert!(
17204            module.exists(),
17205            "failed removal must leave the cgroup intact"
17206        );
17207        assert!(
17208            logs.contains("could not remove module cgroup after process exit; continuing teardown")
17209                && logs.contains("surviving-module"),
17210            "best-effort removal must report the failure without returning it: {logs}"
17211        );
17212    }
17213
17214    #[test]
17215    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17216        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17217        let reason = SuperviseError::Spawn {
17218            program: PathBuf::from("/bin/true"),
17219            source: io::Error::from_raw_os_error(13),
17220            cgroup_path: Some(cgroup_path.clone()),
17221        }
17222        .to_string();
17223
17224        assert!(
17225            reason.contains(&cgroup_path.display().to_string()),
17226            "a pre_exec spawn failure must name the cgroup path: {reason}"
17227        );
17228    }
17229}
17230
17231#[cfg(test)]
17232mod spawn_subscriber_lag_tests {
17233    use super::*;
17234
17235    /// A subscriber whose connection stops draining is dropped once its frame
17236    /// channel fills. The client must learn that from a terminal Error frame
17237    /// after the frames already queued for it, not from a stream that simply
17238    /// goes quiet.
17239    #[tokio::test]
17240    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17241        let feed = SpawnEventFeed::default();
17242        feed.configure_incarnation("lag-incarnation".to_string());
17243        // A one-slot connection queue that nobody reads until the emits are
17244        // done: the forwarder parks on it and the subscriber channel fills.
17245        let (tx, mut rx) = mpsc::channel(1);
17246        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17247            .expect("subscribe");
17248        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17249        for index in 0..emitted {
17250            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17251            // Let the forwarder take what it can so the fill point is the
17252            // subscriber channel, not a scheduling accident.
17253            tokio::task::yield_now().await;
17254        }
17255        assert_eq!(
17256            feed.subscriber_count(),
17257            0,
17258            "the lagged subscriber must be removed"
17259        );
17260
17261        let mut data = Vec::new();
17262        let mut last = None;
17263        loop {
17264            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17265                .await
17266                .expect("the forwarder must finish once the subscriber is dropped");
17267            let Some(outbound) = next else { break };
17268            let frame = outbound.frame;
17269            if frame.header.ty == FrameType::StreamData {
17270                assert!(last.is_none(), "no data may follow the terminal frame");
17271                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17272                data.push(event.cursor.seq);
17273            } else {
17274                assert!(last.is_none(), "exactly one terminal frame");
17275                last = Some(frame);
17276            }
17277        }
17278        assert!(!data.is_empty(), "queued frames drain before the terminal");
17279        for pair in data.windows(2) {
17280            assert_eq!(
17281                pair[1],
17282                pair[0] + 1,
17283                "queued frames arrive dense and in order"
17284            );
17285        }
17286        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17287        assert_eq!(terminal.header.ty, FrameType::Error);
17288        assert_eq!(terminal.header.corr, 7);
17289        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17290        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17291        let detail = body.detail.expect("lagged error carries detail");
17292        assert_eq!(
17293            detail["first_undelivered_cursor"]["seq"],
17294            data.last().unwrap() + 1,
17295            "the named cursor is the first event the subscriber did not receive"
17296        );
17297        assert_eq!(
17298            detail["first_undelivered_cursor"]["daemon_incarnation"],
17299            "lag-incarnation"
17300        );
17301    }
17302}
17303
17304#[cfg(test)]
17305mod terminal_history_read_concurrency_tests {
17306    use super::*;
17307    use crate::terminal_journal::read_pause;
17308    use cortexkit_test_support::ScratchDir;
17309    use std::sync::mpsc as std_mpsc;
17310
17311    fn journaled_ring(
17312        journal: &Arc<crate::terminal_journal::TerminalJournal>,
17313    ) -> Arc<Mutex<TerminalRing>> {
17314        Arc::new(Mutex::new(
17315            TerminalRing::new(TerminalRingConfig::default(), 1)
17316                .with_journal(Some(Arc::clone(journal))),
17317        ))
17318    }
17319
17320    fn crash(at_ms: u64) -> ExitReport {
17321        ExitReport {
17322            kind: ExitKind::Crash,
17323            code: Some(1),
17324            signal: None,
17325            at_ms,
17326        }
17327    }
17328
17329    /// Record an exit on another thread and report whether it finished within
17330    /// `bound`. The recorder thread is left running if it did not.
17331    fn record_within(
17332        module_id: &'static str,
17333        ring: &Arc<Mutex<TerminalRing>>,
17334        at_ms: u64,
17335        bound: Duration,
17336    ) -> bool {
17337        let ring = Arc::clone(ring);
17338        let (done, done_rx) = std_mpsc::channel();
17339        std::thread::spawn(move || {
17340            record_terminal(
17341                module_id,
17342                &ring,
17343                &SpawnEventFeed::default(),
17344                &crash(at_ms),
17345                TerminalDisposition::Restarting,
17346            );
17347            let _ = done.send(());
17348        });
17349        done_rx.recv_timeout(bound).is_ok()
17350    }
17351
17352    /// A history read in progress must not hold the journal writer (which every
17353    /// module's exit recording needs) or the module's own ring. Exits recorded
17354    /// while the read is paused complete promptly; the paused read answers as of
17355    /// the moment it started, and the next read has each exit exactly once.
17356    #[test]
17357    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17358        let dir = ScratchDir::new("terminal-history-concurrent-read");
17359        let path = dir.join("terminals.jsonl");
17360        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17361            path.clone(),
17362            "daemon".into(),
17363        ));
17364        let reader_ring = journaled_ring(&journal);
17365        let other_ring = journaled_ring(&journal);
17366        assert!(record_within(
17367            "reader-module",
17368            &reader_ring,
17369            10,
17370            Duration::from_secs(5)
17371        ));
17372
17373        let (started, release) = read_pause::install(&path);
17374        let reading = {
17375            let ring = Arc::clone(&reader_ring);
17376            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17377        };
17378        started
17379            .recv_timeout(Duration::from_secs(5))
17380            .expect("the history read reached its pause");
17381
17382        let bound = Duration::from_secs(1);
17383        assert!(
17384            record_within("other-module", &other_ring, 20, bound),
17385            "another module's exit waited on a history read (journal writer held)"
17386        );
17387        assert!(
17388            record_within("reader-module", &reader_ring, 30, bound),
17389            "the read module's own exit waited on its history read (ring held)"
17390        );
17391
17392        drop(release);
17393        let paused = reading.join().unwrap();
17394        assert_eq!(
17395            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17396            vec![10],
17397            "an exit recorded after the read began lands in neither half of it"
17398        );
17399        assert_eq!(paused.journal_skipped_lines, 0);
17400        assert_eq!(paused.journal_read_errors, 0);
17401
17402        let after = durable_terminal_history_of(&reader_ring, "reader-module");
17403        assert_eq!(
17404            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17405            vec![10, 30],
17406            "the next read merges ring and journal with no duplicate"
17407        );
17408        assert_eq!(after.journal_skipped_lines, 0);
17409    }
17410}
17411
17412/// What a restart does with the exited process's output readers. These drive
17413/// the same settle functions the supervisor calls, with a reader the test
17414/// holds, so a reader that has not been scheduled by the bound is a controlled
17415/// input rather than something only a loaded machine produces.
17416#[cfg(test)]
17417mod stderr_settle_tests {
17418    use std::{
17419        future::Future,
17420        io,
17421        pin::Pin,
17422        sync::{Arc, Mutex},
17423        task::{Context, Poll},
17424        time::Duration,
17425    };
17426
17427    use tokio::{
17428        io::{AsyncRead, ReadBuf},
17429        sync::oneshot,
17430        time::Instant,
17431    };
17432
17433    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17434    use crate::stderr_tail::{
17435        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17436        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17437    };
17438
17439    const BOUND: Duration = Duration::from_millis(250);
17440
17441    /// Yields `before`, then stays pending until the gate is released, then
17442    /// yields `after` and reaches EOF. The bytes after the gate were written
17443    /// by a process that has already exited; only the reader is behind.
17444    struct HeldReader {
17445        before: Option<Vec<u8>>,
17446        gate: Option<oneshot::Receiver<()>>,
17447        after: io::Cursor<Vec<u8>>,
17448    }
17449
17450    impl AsyncRead for HeldReader {
17451        fn poll_read(
17452            mut self: Pin<&mut Self>,
17453            cx: &mut Context<'_>,
17454            buf: &mut ReadBuf<'_>,
17455        ) -> Poll<io::Result<()>> {
17456            if let Some(bytes) = self.before.take() {
17457                buf.put_slice(&bytes);
17458                return Poll::Ready(Ok(()));
17459            }
17460            if let Some(gate) = self.gate.as_mut() {
17461                match Pin::new(gate).poll(cx) {
17462                    Poll::Pending => return Poll::Pending,
17463                    Poll::Ready(_) => self.gate = None,
17464                }
17465            }
17466            Pin::new(&mut self.after).poll_read(cx, buf)
17467        }
17468    }
17469
17470    struct DiscardSink;
17471
17472    impl OutputSink for DiscardSink {
17473        fn write_line(&mut self, _line: &[u8]) {}
17474    }
17475
17476    fn line(text: &str) -> TailEntry {
17477        TailEntry::Line {
17478            text: text.to_string(),
17479            truncated: false,
17480            at_ms: None,
17481        }
17482    }
17483
17484    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17485        ring.lock().unwrap()
17486    }
17487
17488    /// Start a reader for a new process generation that delivers `before`
17489    /// immediately and `after` only once the returned sender fires (or is
17490    /// dropped).
17491    fn held_pump(
17492        ring: &Arc<Mutex<StderrRing>>,
17493        before: &str,
17494        after: &str,
17495    ) -> (StderrPump, oneshot::Sender<()>) {
17496        let generation = lock(ring).begin_process();
17497        let (release, gate) = oneshot::channel();
17498        let reader = HeldReader {
17499            before: Some(before.as_bytes().to_vec()),
17500            gate: Some(gate),
17501            after: io::Cursor::new(after.as_bytes().to_vec()),
17502        };
17503        let task = tokio::spawn(pump_stderr_to(
17504            reader,
17505            Arc::clone(ring),
17506            generation,
17507            DiscardSink,
17508        ));
17509        (StderrPump { task, generation }, release)
17510    }
17511
17512    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17513        for _ in 0..1000 {
17514            if done(&lock(ring)) {
17515                return;
17516            }
17517            tokio::time::sleep(Duration::from_millis(1)).await;
17518        }
17519        panic!(
17520            "ring never reached the expected state: {:?}",
17521            lock(ring).snapshot(None, None)
17522        );
17523    }
17524
17525    #[tokio::test(start_paused = true)]
17526    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17527        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17528        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17529
17530        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17531        let before_release = lock(&ring).snapshot(None, None);
17532        assert!(
17533            matches!(before_release.capture, CaptureState::Incomplete { .. }),
17534            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17535        );
17536
17537        // The restart: the next process starts and writes before the old
17538        // reader catches up.
17539        let next = lock(&ring).begin_process();
17540        lock(&ring).push_line_from(next, "next process booting");
17541        release.send(()).unwrap();
17542        wait_until(&ring, |ring| {
17543            ring.snapshot(None, None).capture == CaptureState::Captured
17544        })
17545        .await;
17546
17547        assert_eq!(
17548            untimed(lock(&ring).snapshot(None, None).entries),
17549            vec![
17550                line("booting"),
17551                line("config error: missing storage"),
17552                TailEntry::ProcessStart,
17553                line("next process booting"),
17554            ],
17555            "the crash's last line must survive a slow reader and stay in the crashed process's section"
17556        );
17557    }
17558
17559    #[tokio::test(start_paused = true)]
17560    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17561    ) {
17562        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17563        // `_held` is never fired: a descendant keeps the pipe open for the
17564        // whole test.
17565        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17566
17567        let started = Instant::now();
17568        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17569        assert_eq!(
17570            started.elapsed(),
17571            BOUND,
17572            "the restart must wait exactly the bound for a pipe that stays open, no longer"
17573        );
17574
17575        let next = lock(&ring).begin_process();
17576        lock(&ring).push_line_from(next, "next process booting");
17577        tokio::time::sleep(Duration::from_secs(60)).await;
17578
17579        let snapshot = lock(&ring).snapshot(None, None);
17580        match &snapshot.capture {
17581            CaptureState::Incomplete { reason } => assert!(
17582                reason.contains("had not reached EOF") && reason.contains("250ms"),
17583                "the reason must say what is missing and after how long: {reason}"
17584            ),
17585            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17586        }
17587        assert_eq!(
17588            untimed(snapshot.entries),
17589            vec![
17590                line("parent exiting"),
17591                TailEntry::ProcessStart,
17592                line("next process booting"),
17593            ]
17594        );
17595    }
17596
17597    #[tokio::test(start_paused = true)]
17598    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17599        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17600        let (pump, release) = held_pump(&ring, "one\n", "two\n");
17601        release.send(()).unwrap();
17602
17603        settle_stderr_pump("clean", &ring, pump, BOUND).await;
17604
17605        let snapshot = lock(&ring).snapshot(None, None);
17606        assert_eq!(snapshot.capture, CaptureState::Captured);
17607        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17608    }
17609
17610    #[tokio::test(start_paused = true)]
17611    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17612        const BURST: usize = 150;
17613        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17614        let path = capture.join("two-pipe.stderr.log");
17615        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17616        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17617        let generation = lock(&ring).begin_process();
17618        let stdout = (0..BURST)
17619            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17620            .collect::<String>();
17621        let stderr = (0..BURST)
17622            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17623            .collect::<String>();
17624        // Model a child that has written both bursts and exited. Hold the stdout
17625        // reader after one line and a partial prefix, as if pipe readiness or
17626        // capture-file writes had delayed it. The final line also needs EOF to
17627        // flush: neither complete nor partial buffered output may be discarded.
17628        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17629        let (release, gate) = oneshot::channel();
17630        let reader = HeldReader {
17631            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17632            gate: Some(gate),
17633            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17634        };
17635        let (finished, completion) = oneshot::channel();
17636        let stdout_sink = sink.clone();
17637        let pump = tokio::spawn(async move {
17638            pump_stdout_to(reader, stdout_sink).await;
17639            finished.send(()).unwrap();
17640        });
17641        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17642
17643        let started = Instant::now();
17644        settle_stdout_pump("two-pipe", pump, BOUND).await;
17645        assert_eq!(
17646            started.elapsed(),
17647            BOUND,
17648            "a late reader must not hold up restart"
17649        );
17650        assert_eq!(
17651            std::fs::read_to_string(&path).unwrap().lines().count(),
17652            BURST + 1
17653        );
17654
17655        release
17656            .send(())
17657            .expect("stdout reader must survive the exit drain bound");
17658        tokio::time::timeout(Duration::from_secs(1), completion)
17659            .await
17660            .expect("released stdout reader must reach EOF")
17661            .expect("stdout capture must finish at EOF, not be cancelled");
17662        let contents = std::fs::read_to_string(&path).unwrap();
17663        let mut lines = contents
17664            .lines()
17665            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17666            .collect::<Vec<_>>();
17667        let mut expected = (0..BURST)
17668            .flat_map(|i| {
17669                [
17670                    format!("out-{i:04}-{}", "o".repeat(64)),
17671                    format!("err-{i:04}-{}", "e".repeat(64)),
17672                ]
17673            })
17674            .collect::<Vec<_>>();
17675        lines.sort_unstable();
17676        expected.sort_unstable();
17677        assert_eq!(
17678            lines, expected,
17679            "both pipes must retain every complete line exactly once"
17680        );
17681    }
17682}
17683
17684/// Containment of a module's process tree (issue #109).
17685///
17686/// The behaviour these defend against is a module helper surviving its module:
17687/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
17688/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
17689/// compounds it.
17690///
17691/// They run against the SUPERVISOR rather than the job-object crate because the
17692/// claim is about teardown: a crate-level test proves a job can reap a tree, not
17693/// that the daemon's drain path reaches it.
17694///
17695/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
17696/// lane there is a separate containment path with its own tests.
17697#[cfg(all(test, windows))]
17698mod job_containment_tests {
17699    use super::*;
17700    use cortexkit_test_support::ScratchDir;
17701    use std::{
17702        path::{Path, PathBuf},
17703        sync::{Arc, Mutex},
17704        time::{Duration, Instant},
17705    };
17706
17707    /// The stub, expected beside this test executable.
17708    ///
17709    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17710    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17711    /// failure then reads as a broken test rather than an unbuilt dependency.
17712    fn stub_path() -> PathBuf {
17713        let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
17714        assert!(
17715            path.exists(),
17716            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17717             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17718            path.display()
17719        );
17720        path
17721    }
17722
17723    /// Poll for the grandchild pid the stub records, and parse it.
17724    fn read_grandchild_pid(path: &Path) -> u32 {
17725        let deadline = Instant::now() + Duration::from_secs(10);
17726        loop {
17727            if let Ok(contents) = std::fs::read_to_string(path) {
17728                if let Ok(pid) = contents.trim().parse() {
17729                    return pid;
17730                }
17731            }
17732            assert!(
17733                Instant::now() < deadline,
17734                "the stub never recorded a grandchild pid at {}",
17735                path.display()
17736            );
17737            std::thread::sleep(Duration::from_millis(10));
17738        }
17739    }
17740
17741    /// Everything one fixture run needs, so the two tests below differ in exactly
17742    /// one place: whether the child is contained.
17743    struct Fixture {
17744        _dir: ScratchDir,
17745        module_id: String,
17746        grandchild: u32,
17747        child: Option<SupervisedChild>,
17748        registry: Arc<Registry>,
17749        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17750        terminal_ring: Arc<Mutex<TerminalRing>>,
17751        spawn_events: SpawnEventFeed,
17752    }
17753
17754    fn fixture(label: &str, module_id: &str) -> Fixture {
17755        let dir = ScratchDir::new(label);
17756        let pid_file = dir.join("grandchild.pid");
17757        let supervisor = Supervisor::new_for_test(
17758            Arc::new(Registry::default()),
17759            RestartPolicy::new(3, Duration::ZERO),
17760        );
17761        let runtime = supervisor.runtime_config();
17762        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17763        let spec = ModuleSpec {
17764            module_id: module_id.to_string(),
17765            program: stub_path(),
17766            // Zero args deliberately: a `--subc` argument would make the stub dial
17767            // a daemon that is not there, and the failure would land in the same
17768            // stderr ring this fixture exists to keep quiet.
17769            args: Vec::new(),
17770            env: vec![
17771                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17772                (
17773                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17774                    pid_file.display().to_string(),
17775                ),
17776            ],
17777            reserved: false,
17778            reserved_prefixes: Vec::new(),
17779            protocol: ModuleProtocol::Subc,
17780            overlap: Default::default(),
17781            carry_containment: false,
17782        };
17783        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17784            .expect("spawn the supervised fixture");
17785        let grandchild = read_grandchild_pid(&pid_file);
17786        Fixture {
17787            _dir: dir,
17788            module_id: module_id.to_string(),
17789            grandchild,
17790            child: Some(child),
17791            registry: Arc::new(Registry::default()),
17792            snapshot,
17793            terminal_ring: Arc::clone(&runtime.terminal_ring),
17794            spawn_events: SpawnEventFeed::default(),
17795        }
17796    }
17797
17798    impl Fixture {
17799        /// Drain through the supervisor's own teardown path.
17800        async fn drain(&mut self) {
17801            let child = self
17802                .child
17803                .take()
17804                .expect("the fixture child is still present");
17805            drain_child_to_state(
17806                &self.module_id,
17807                ModuleProtocol::Subc,
17808                // No forwarding table in this fixture, so nothing reaches the
17809                // child over a connection.
17810                StopNotice::NotSent,
17811                &self.registry,
17812                None,
17813                &self.snapshot,
17814                &self.terminal_ring,
17815                &self.spawn_events,
17816                child,
17817                Duration::from_millis(500),
17818                ModuleState::Stopped,
17819                Some(false),
17820                &FailureContext::operator(),
17821            )
17822            .await
17823            .expect("drain the supervised fixture");
17824        }
17825    }
17826
17827    /// Teardown reaps the grandchild, not merely the direct child.
17828    ///
17829    /// This is the assertion the change exists for. Before containment the
17830    /// grandchild survived: it is a separate process, and `start_kill` is
17831    /// `TerminateProcess` scoped to one pid.
17832    #[tokio::test]
17833    async fn teardown_reaps_the_grandchild() {
17834        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17835        let grandchild = fixture.grandchild;
17836
17837        assert!(
17838            subc_jobobject::process_exists(grandchild),
17839            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17840        );
17841
17842        fixture.drain().await;
17843
17844        assert!(
17845            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17846            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17847        );
17848    }
17849
17850    /// The mutation control: with containment withheld, the grandchild survives
17851    /// the same kill.
17852    ///
17853    /// This is the defect reproduction from #109 — a direct-child kill reaches
17854    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17855    /// supervisor because `spawn_and_mark_running` now always contains on
17856    /// Windows, which is the point: there is no longer a path that spawns
17857    /// uncontained, so the control has to construct one.
17858    ///
17859    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17860    /// grandchild ever dies here, that test is passing for a reason unrelated to
17861    /// the job object and the containment claim is unproven.
17862    #[test]
17863    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17864        let dir = ScratchDir::new("teardown-uncontained");
17865        let pid_file = dir.join("grandchild.pid");
17866        let mut child = std::process::Command::new(stub_path())
17867            .env("FAKE_AFT_NEVER_CONNECT", "1")
17868            .env(
17869                "FAKE_AFT_GRANDCHILD_PID_FILE",
17870                pid_file.display().to_string(),
17871            )
17872            .stdin(std::process::Stdio::null())
17873            .stdout(std::process::Stdio::null())
17874            .stderr(std::process::Stdio::null())
17875            .spawn()
17876            .expect("spawn the uncontained fixture");
17877        let grandchild = read_grandchild_pid(&pid_file);
17878
17879        // Exactly what the pre-fix teardown did: kill the direct child.
17880        child.kill().expect("kill the direct child");
17881        let _ = child.wait();
17882
17883        assert!(
17884            subc_jobobject::process_exists(grandchild),
17885            "grandchild {grandchild} died with the direct child, so this control no longer \
17886             distinguishes contained from uncontained teardown and the regression test is \
17887             passing vacuously"
17888        );
17889
17890        // The orphan this control demonstrates is the leak the fix prevents, so
17891        // the control must not leave one behind.
17892        kill_tree(grandchild);
17893    }
17894
17895    /// Crash durability: closing the containment handle reaps the tree with no
17896    /// teardown code running at all.
17897    ///
17898    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17899    /// call anything — and it is why containment is a kernel property of the
17900    /// handle rather than a step in the drain. Discovered by getting the
17901    /// mutation control wrong: clearing `job` to "disable" containment instead
17902    /// killed the tree, which is the guarantee, not a mistake.
17903    #[tokio::test]
17904    async fn dropping_containment_reaps_the_grandchild() {
17905        let mut fixture = fixture("drop-containment", "tree-drop");
17906        let grandchild = fixture.grandchild;
17907
17908        assert!(subc_jobobject::process_exists(grandchild));
17909
17910        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17911        fixture.child.as_mut().expect("child present").job = None;
17912
17913        assert!(
17914            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17915            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17916             crash would leave the tree behind"
17917        );
17918    }
17919
17920    /// Kill a pid and its tree, then confirm it is gone.
17921    fn kill_tree(pid: u32) {
17922        let _ = std::process::Command::new("taskkill.exe")
17923            .args(["/PID", &pid.to_string(), "/T", "/F"])
17924            .stdin(std::process::Stdio::null())
17925            .stdout(std::process::Stdio::null())
17926            .stderr(std::process::Stdio::null())
17927            .status();
17928        assert!(
17929            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17930            "could not clean up grandchild {pid}"
17931        );
17932    }
17933}
17934
17935#[cfg(test)]
17936mod privacy_trampoline_configuration_tests {
17937    #[tokio::test]
17938    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17939    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17940        #[cfg(target_os = "macos")]
17941        {
17942            let supervisor = super::Supervisor::new(
17943                std::sync::Arc::new(crate::Registry::default()),
17944                super::RestartPolicy::default(),
17945            );
17946            let error = supervisor.spawn(spec()).unwrap_err();
17947            assert!(
17948                error
17949                    .to_string()
17950                    .contains("no privacy trampoline configured"),
17951                "{error}"
17952            );
17953        }
17954    }
17955
17956    #[tokio::test]
17957    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17958    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17959        #[cfg(target_os = "macos")]
17960        {
17961            let supervisor = super::Supervisor::new(
17962                std::sync::Arc::new(crate::Registry::default()),
17963                super::RestartPolicy::default(),
17964            )
17965            .with_privacy_trampoline(std::env::current_exe().unwrap());
17966            let error = supervisor.spawn(spec()).unwrap_err();
17967            assert!(
17968                error
17969                    .to_string()
17970                    .contains("binary does not implement the privacy trampoline protocol"),
17971                "{error}"
17972            );
17973        }
17974    }
17975
17976    #[cfg(target_os = "macos")]
17977    fn spec() -> super::ModuleSpec {
17978        super::ModuleSpec {
17979            module_id: "privacy-configuration".into(),
17980            program: "/bin/sleep".into(),
17981            args: vec!["30".into()],
17982            env: vec![],
17983            reserved: false,
17984            reserved_prefixes: vec![],
17985            protocol: subc_control::ModuleProtocol::None,
17986            overlap: super::ModuleOverlap::Exclusive,
17987            carry_containment: false,
17988        }
17989    }
17990}
17991
17992#[cfg(test)]
17993mod privacy_exec_boundary_tests {
17994    #[cfg(target_os = "macos")]
17995    use super::*;
17996    #[cfg(target_os = "macos")]
17997    use std::{
17998        io::{Read, Write},
17999        net::{TcpListener, TcpStream},
18000    };
18001
18002    /// Unit-test-only pause at the actual early image read, not at a later
18003    /// status read. Production supervisors never inspect this environment key.
18004    #[cfg(target_os = "macos")]
18005    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18006        if let Some((_, path)) = spec
18007            .env
18008            .iter()
18009            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18010        {
18011            let mut barrier = TcpStream::connect(path).unwrap();
18012            barrier
18013                .set_read_timeout(Some(Duration::from_secs(30)))
18014                .unwrap();
18015            barrier.write_all(&pid.to_ne_bytes()).unwrap();
18016            let mut release = [0];
18017            barrier.read_exact(&mut release).unwrap();
18018            assert_eq!(&release, b"X");
18019        }
18020    }
18021
18022    #[cfg(target_os = "macos")]
18023    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18024        ModuleSpec {
18025            module_id: "privacy-boundary".into(),
18026            program: program.into(),
18027            args: args.iter().map(|arg| (*arg).into()).collect(),
18028            env: vec![],
18029            reserved: false,
18030            reserved_prefixes: vec![],
18031            protocol: ModuleProtocol::None,
18032            overlap: ModuleOverlap::Exclusive,
18033            carry_containment: false,
18034        }
18035    }
18036
18037    #[cfg(target_os = "macos")]
18038    async fn accept(listener: TcpListener) -> TcpStream {
18039        // Socket readiness, not elapsed time, establishes both pause points.
18040        let listener = tokio::net::TcpListener::from_std({
18041            listener.set_nonblocking(true).unwrap();
18042            listener
18043        })
18044        .unwrap();
18045        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18046            .await
18047            .unwrap()
18048            .unwrap();
18049        let stream = stream.into_std().unwrap();
18050        stream.set_nonblocking(false).unwrap();
18051        stream
18052            .set_read_timeout(Some(Duration::from_secs(30)))
18053            .unwrap();
18054        stream
18055    }
18056
18057    #[tokio::test]
18058    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18059    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18060        #[cfg(target_os = "macos")]
18061        {
18062            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18063            // Loopback sockets also work when the replay adapter's TMPDIR is
18064            // longer than Darwin's Unix-domain socket path limit.
18065            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18066            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18067            let record = root.join("live-children.json");
18068            let supervisor =
18069                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18070                    .with_live_children_record(&record);
18071            let runtime = supervisor.runtime_config();
18072            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18073            let mut spec = spec("/bin/sleep", &["30"]);
18074            spec.env = vec![
18075                (
18076                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18077                    exec_listener.local_addr().unwrap().to_string(),
18078                ),
18079                (
18080                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18081                    sample_listener.local_addr().unwrap().to_string(),
18082                ),
18083            ];
18084            let spawn = tokio::task::spawn_blocking(move || {
18085                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18086            });
18087            let mut sample = accept(sample_listener).await;
18088            let mut pid = [0; 4];
18089            sample.read_exact(&mut pid).unwrap();
18090            let pid = u32::from_ne_bytes(pid);
18091            let mut exec = accept(exec_listener).await;
18092            let mut ready = [0];
18093            exec.read_exact(&mut ready).unwrap();
18094            assert_eq!(&ready, b"R");
18095            // The early read is guaranteed to see a real, non-null trampoline
18096            // image: the fixture has reached its barrier and cannot exec yet.
18097            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18098            assert_eq!(
18099                observe_spawned_image(pid).unwrap().executable,
18100                Some(trampoline)
18101            );
18102            sample.write_all(b"X").unwrap();
18103            let mut child = spawn.await.unwrap();
18104            let early = crate::live_children::read_record(&record).unwrap();
18105            assert_eq!(early.len(), 1);
18106            assert_eq!(early[0].pid, pid);
18107            assert_eq!(
18108                early[0].executable, None,
18109                "unconfirmed trampoline image entered the roster"
18110            );
18111            assert!(child.report_ready.get().is_none());
18112            // The barrier's duration is unrelated to the production five-second
18113            // exec budget. Start the test's confirmation budget upon release.
18114            child.privacy_exec.as_mut().unwrap().deadline =
18115                tokio::time::Instant::now() + Duration::from_secs(30);
18116            exec.write_all(b"X").unwrap();
18117            child.confirm_privacy_exec().await;
18118            assert_eq!(child.spawn_failure, None);
18119            assert!(child.report_ready.get().is_some());
18120            let confirmed = crate::live_children::read_record(&record).unwrap();
18121            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18122            assert_ne!(module, trampoline);
18123            assert_eq!(confirmed[0].executable, Some(module.into()));
18124            child.start_kill().unwrap();
18125            child.wait().await.unwrap();
18126            child.release_roster();
18127        }
18128    }
18129
18130    #[tokio::test]
18131    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18132    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18133        #[cfg(target_os = "macos")]
18134        {
18135            let registry = Arc::new(Registry::default());
18136            let policy = RestartPolicy::new(0, Duration::ZERO);
18137            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
18138            let runtime = supervisor.runtime_config();
18139            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18140            let spec = spec("/bin/sh", &["-c", "exit 121"]);
18141            // Drive spawn and confirmation separately instead of starting the
18142            // monitor. WNOWAIT observes a real exit without consuming its status,
18143            // so confirmation's first try_wait must take the already-exited arm.
18144            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18145            let pid = child.pid;
18146            tokio::task::spawn_blocking(move || {
18147                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18148            })
18149            .await
18150            .unwrap()
18151            .unwrap();
18152            child.privacy_exec.as_mut().unwrap().deadline =
18153                tokio::time::Instant::now() + Duration::from_secs(30);
18154            let status = child.wait().await.unwrap();
18155            assert_eq!(status.code(), Some(121));
18156            assert!(child.privacy_exec.is_none());
18157            assert!(
18158                child.report_ready.get().is_none(),
18159                "an exited module must not publish a live pid"
18160            );
18161            let report = classify_reaped_child_exit(&snapshot, &child, &status);
18162            on_child_exit(
18163                &spec,
18164                policy,
18165                &registry,
18166                &snapshot,
18167                &runtime.terminal_ring,
18168                &runtime.spawn_events,
18169                &runtime.child_roster,
18170                report,
18171                true,
18172            )
18173            .await;
18174            let state = lock_snapshot(&snapshot).unwrap();
18175            assert_eq!(state.state, ModuleState::Failed);
18176            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18177            assert_eq!(state.reported_pid(), None);
18178            drop(state);
18179            let history = runtime.terminal_ring.lock().unwrap().snapshot();
18180            assert_eq!(history.entries.len(), 1);
18181            let terminal = &history.entries[0];
18182            assert_eq!(terminal.exit_code, Some(121));
18183            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18184            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18185            assert_eq!(
18186                terminal.disposition_detail.as_deref(),
18187                Some(policy.budget_exhausted_detail().as_str()),
18188                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18189            );
18190            assert_eq!(child.spawn_failure, None);
18191            child.release_roster();
18192        }
18193    }
18194}
18195
18196/// The daemon's real spawn path hands a subc-wire child its launch nonce on
18197/// descriptor 3, without an environment copy. The shell records the nonce
18198/// and its environment after exec so these tests observe the real handover.
18199#[cfg(all(test, unix))]
18200mod launch_nonce_descriptor_tests {
18201    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18202    use crate::stderr_tail::{StderrRing, StderrTailConfig};
18203    use cortexkit_test_support::ScratchDir;
18204    use std::{
18205        path::PathBuf,
18206        sync::{Arc, Mutex},
18207        time::{Duration, Instant},
18208    };
18209
18210    async fn probe(role: super::SpawnRole) {
18211        let scratch = ScratchDir::new("launch-nonce-descriptor");
18212        let fd_copy = scratch.join("from-descriptor");
18213        let env_copy = scratch.join("environment");
18214        let script = format!(
18215            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18216            fd = fd_copy.display(), env = env_copy.display(),
18217        );
18218        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18219        let spec = ModuleSpec {
18220            module_id: "nonce-descriptor-probe".to_string(),
18221            program: PathBuf::from("/bin/sh"),
18222            args: vec!["-c".to_string(), script],
18223            env: vec![
18224                xdg("XDG_DATA_HOME"),
18225                xdg("XDG_RUNTIME_DIR"),
18226                xdg("XDG_CONFIG_HOME"),
18227                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18228            ],
18229            reserved: true,
18230            reserved_prefixes: Vec::new(),
18231            protocol: ModuleProtocol::Subc,
18232            overlap: Default::default(),
18233            carry_containment: false,
18234        };
18235        let handle = SupervisorHandle::new();
18236        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18237        let roster = ChildRoster::default();
18238        #[cfg(target_os = "macos")]
18239        {
18240            let path = super::test_privacy_trampoline();
18241            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18242        }
18243        let child = super::spawn_child_in_slot(
18244            &spec,
18245            None,
18246            Some(&handle),
18247            &ring,
18248            None,
18249            &roster,
18250            #[cfg(target_os = "linux")]
18251            None,
18252            role,
18253            matches!(role, super::SpawnRole::SwapCandidate),
18254        )
18255        .expect("spawn probe");
18256        let deadline = Instant::now() + Duration::from_secs(10);
18257        while !(fd_copy.exists() && env_copy.exists()) {
18258            assert!(Instant::now() < deadline, "probe never wrote its copies");
18259            tokio::time::sleep(Duration::from_millis(20)).await;
18260        }
18261        let nonce = std::fs::read_to_string(fd_copy).unwrap();
18262        assert!(!nonce.is_empty());
18263        let environment = std::fs::read_to_string(env_copy).unwrap();
18264        assert!(environment
18265            .lines()
18266            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18267        let copy = environment
18268            .lines()
18269            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18270        assert_eq!(
18271            copy, None,
18272            "Unix children must never receive the environment nonce"
18273        );
18274        if matches!(role, super::SpawnRole::Plain) {
18275            assert_eq!(
18276                handle.spawn_nonce(&spec.module_id).as_deref(),
18277                Some(nonce.as_str())
18278            );
18279        }
18280        drop(child);
18281    }
18282
18283    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18284    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18285        probe(super::SpawnRole::Plain).await;
18286    }
18287
18288    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18289    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18290        probe(super::SpawnRole::SwapCandidate).await;
18291    }
18292}
18293
18294#[cfg(all(test, target_os = "linux"))]
18295#[path = "supervise/cgroup_reap_tests.rs"]
18296mod cgroup_reap_tests;
18297
18298#[cfg(all(test, target_os = "linux"))]
18299mod cgroup_containment_tests {
18300    use super::*;
18301    use cortexkit_test_support::ScratchDir;
18302
18303    #[tokio::test]
18304    async fn linux_subtree_memory_covers_module_children() {
18305        let dir = ScratchDir::new("subtree-memory");
18306        let root = PathBuf::from(format!(
18307            "/sys/fs/cgroup/subc-memory-test-{}-{}",
18308            std::process::id(),
18309            unix_ms_now()
18310        ));
18311        if let Err(error) = std::fs::create_dir(&root) {
18312            assert!(
18313                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18314                "required cgroup test cannot execute: {error}"
18315            );
18316            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18317            return;
18318        }
18319        let placement = subc_cgroup::prepare_at(&root)
18320            .unwrap()
18321            .expect("delegated cgroup");
18322        let supervisor = Supervisor::new_for_test(
18323            Arc::new(Registry::default()),
18324            RestartPolicy::new(3, Duration::ZERO),
18325        )
18326        .with_cgroup_placement(Some(placement));
18327        let ready = dir.join("ready");
18328        let module = supervisor
18329            .spawn(ModuleSpec {
18330                module_id: "memory-tree".into(),
18331                program: PathBuf::from("/bin/sh"),
18332                args: vec![
18333                    "-c".into(),
18334                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18335                    "fixture".into(),
18336                    ready.display().to_string(),
18337                ],
18338                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18339                    .into_iter()
18340                    .map(|key| (key.into(), dir.display().to_string()))
18341                    .collect(),
18342                reserved: false,
18343                reserved_prefixes: Vec::new(),
18344                protocol: ModuleProtocol::None,
18345                overlap: Default::default(),
18346                carry_containment: false,
18347            })
18348            .unwrap();
18349        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18350        while !ready.exists() {
18351            assert!(
18352                tokio::time::Instant::now() < deadline,
18353                "children never started"
18354            );
18355            tokio::time::sleep(Duration::from_millis(10)).await;
18356        }
18357        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18358        let reading = &readings[0];
18359        assert!(
18360            reading.process_count.is_some_and(|count| count >= 4),
18361            "{reading:?}"
18362        );
18363        assert!(
18364            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18365            "{reading:?}"
18366        );
18367        module.stop().await.unwrap();
18368        drop(module);
18369        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18370        std::fs::remove_dir(&root).unwrap();
18371        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18372    }
18373
18374    fn running(pid: u32) -> bool {
18375        // An orphan can remain a zombie until the container init reaps it.
18376        std::fs::read_to_string(format!("/proc/{pid}/stat"))
18377            .ok()
18378            .and_then(|stat| {
18379                stat.rsplit_once(") ")
18380                    .map(|(_, rest)| rest.starts_with('Z'))
18381            })
18382            .is_some_and(|zombie| !zombie)
18383    }
18384
18385    #[tokio::test]
18386    async fn linux_teardown_reaps_the_grandchild() {
18387        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18388    }
18389
18390    #[tokio::test]
18391    async fn linux_shutdown_straggler_reaps_the_grandchild() {
18392        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18393    }
18394
18395    async fn teardown_tree(test_name: &str, shutdown: bool) {
18396        let dir = ScratchDir::new(test_name);
18397        let root = PathBuf::from(format!(
18398            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18399            std::process::id(),
18400            unix_ms_now()
18401        ));
18402        if let Err(error) = std::fs::create_dir(&root) {
18403            assert!(
18404                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18405                "required cgroup test cannot execute: {error}"
18406            );
18407            eprintln!(
18408                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18409                root.display()
18410            );
18411            return;
18412        }
18413        let placement = subc_cgroup::prepare_at(&root)
18414            .expect("prepare isolated kernel cgroup")
18415            .expect("isolated cgroup is delegated");
18416        let module_id = "tree-teardown";
18417        let module = placement
18418            .module_path(module_id)
18419            .expect("create isolated module cgroup");
18420        if !module.join("cgroup.kill").exists() {
18421            std::fs::remove_dir(&module).unwrap();
18422            std::fs::remove_dir(root.join("subc-modules")).unwrap();
18423            std::fs::remove_dir(&root).unwrap();
18424            assert!(
18425                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18426                "required cgroup.kill interface unavailable"
18427            );
18428            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18429            return;
18430        }
18431        let supervisor = Supervisor::new_for_test(
18432            Arc::new(Registry::default()),
18433            RestartPolicy::new(3, Duration::ZERO),
18434        )
18435        .with_cgroup_placement(Some(placement));
18436        let mut runtime = supervisor.runtime_config();
18437        runtime.child_roster = runtime
18438            .child_roster
18439            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18440        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18441        let pid_file = dir.join("grandchild.pid");
18442        let spec = ModuleSpec {
18443            module_id: module_id.to_string(),
18444            program: PathBuf::from("/bin/sh"),
18445            args: vec![
18446                "-c".into(),
18447                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18448                "fixture".into(),
18449                pid_file.display().to_string(),
18450            ],
18451            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18452                .into_iter()
18453                .map(|key| (key.to_string(), dir.display().to_string()))
18454                .collect(),
18455            reserved: false,
18456            reserved_prefixes: Vec::new(),
18457            protocol: ModuleProtocol::None,
18458            overlap: Default::default(),
18459            carry_containment: false,
18460        };
18461        let child =
18462            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18463        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18464        let grandchild: u32 = loop {
18465            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18466                if let Ok(pid) = contents.trim().parse() {
18467                    break pid;
18468                }
18469            }
18470            assert!(
18471                tokio::time::Instant::now() < deadline,
18472                "grandchild pid was not recorded"
18473            );
18474            tokio::time::sleep(Duration::from_millis(10)).await;
18475        };
18476        assert!(
18477            running(grandchild),
18478            "grandchild must be alive before teardown"
18479        );
18480        if shutdown {
18481            let mut child = child;
18482            crate::child_roster::end_children_for_daemon_shutdown(
18483                &runtime.child_roster,
18484                false,
18485                std::future::pending(),
18486            )
18487            .await;
18488            child.wait().await.expect("reap shutdown straggler");
18489        } else {
18490            drain_child_to_state(
18491                module_id,
18492                ModuleProtocol::None,
18493                StopNotice::NotSent,
18494                &Registry::default(),
18495                None,
18496                &snapshot,
18497                &runtime.terminal_ring,
18498                &SpawnEventFeed::default(),
18499                child,
18500                Duration::from_millis(100),
18501                ModuleState::Stopped,
18502                Some(false),
18503                &FailureContext::operator(),
18504            )
18505            .await
18506            .expect("real supervisor teardown");
18507        }
18508        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18509        while running(grandchild) && tokio::time::Instant::now() < deadline {
18510            tokio::time::sleep(Duration::from_millis(10)).await;
18511        }
18512        let survived = running(grandchild);
18513        // Kill a surviving grandchild so a failed test does not leave it behind.
18514        if survived {
18515            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18516            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18517            tokio::time::sleep(Duration::from_millis(100)).await;
18518        }
18519        if module.exists() {
18520            std::fs::remove_dir(&module).expect("remove empty module cgroup");
18521        }
18522        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18523        std::fs::remove_dir(&root).unwrap();
18524        assert!(
18525            !survived,
18526            "grandchild {grandchild} outlived module teardown"
18527        );
18528        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18529    }
18530}
18531
18532/// A slow live-children record write on the restart path must not stop the
18533/// daemon from answering the replacement module's authentication handshake.
18534#[cfg(all(test, unix))]
18535mod record_write_stall_tests {
18536    use super::*;
18537    use std::sync::atomic::{AtomicBool, Ordering};
18538    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18539
18540    /// `subc-client-rs` gives the whole handshake this long before the module
18541    /// exits with `Timeout { stage: ServerProof }`.
18542    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18543    /// How long the replacement's admission write is held: longer than the
18544    /// module's deadline, standing in for the delay a busy disk can add
18545    /// across the two syncs in that write.
18546    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18547
18548    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
18549    /// On macOS every module is launched through the privacy trampoline, and
18550    /// a module that is not a `#!` script is admitted only if its running
18551    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
18552    /// the shell soon replaces itself with `/bin/sleep`, which does not match
18553    /// the configured `/bin/sh`. The image is read only after the spawn's
18554    /// live-children record write, and here that write lasted long enough for
18555    /// the shell to have exec'd every time, so each process was refused and
18556    /// killed right after spawn, and the supervisor crash-looped it with
18557    /// backoff until its restart budget ran out. The restart this test
18558    /// measures would then never produce a module at all.
18559    fn sleeping_spec() -> ModuleSpec {
18560        ModuleSpec {
18561            module_id: "record-write-stall".into(),
18562            program: "/bin/sleep".into(),
18563            args: vec!["600".into()],
18564            env: vec![],
18565            reserved: false,
18566            reserved_prefixes: vec![],
18567            protocol: ModuleProtocol::None,
18568            overlap: Default::default(),
18569            carry_containment: false,
18570        }
18571    }
18572
18573    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18574    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18575        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18576        let supervisor =
18577            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18578                .with_live_children_record(home.join("live-children.json"));
18579        let module = supervisor.spawn(sleeping_spec()).unwrap();
18580        // `reported_pid` names a process only once it is confirmed as the
18581        // module (on macOS, once the trampoline's image check has passed), so
18582        // a process the daemon is about to refuse and kill never counts as
18583        // the running module, here or in the final wait below.
18584        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18585            loop {
18586                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18587                    .unwrap()
18588                    .reported_pid()
18589                {
18590                    return pid;
18591                }
18592                tokio::time::sleep(Duration::from_millis(10)).await;
18593            }
18594        })
18595        .await
18596        .expect("the first process was admitted as the module");
18597
18598        // The daemon's listener, served by this runtime's workers like the
18599        // supervisor's own tasks.
18600        let key = vec![0x42; 32];
18601        let daemon_id = [0x24; 16];
18602        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18603        let port = listener.local_addr().unwrap().port();
18604        let server = tokio::spawn(crate::server::serve_listener(
18605            listener,
18606            Arc::new(crate::router::Router::with_default_self_handler()),
18607            crate::server::ServerAuth::with_limits(
18608                key.clone(),
18609                daemon_id,
18610                "test",
18611                crate::server::DEFAULT_AUTH_DEADLINE,
18612                4,
18613            ),
18614        ));
18615
18616        // Hold the write that admits the replacement, once, and say when it
18617        // has started.
18618        let (held_tx, held_rx) = std::sync::mpsc::channel();
18619        let armed = AtomicBool::new(true);
18620        supervisor
18621            .child_roster
18622            .set_record_write_hook_for_test(move |live| {
18623                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18624                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18625                    let _ = held_tx.send(());
18626                    std::thread::sleep(HELD_RECORD_WRITE);
18627                }
18628            });
18629
18630        // The replacement module, as a separate process would be: its own
18631        // thread and runtime, so only the daemon's runtime can be stalled.
18632        let module_process = std::thread::spawn(move || {
18633            held_rx
18634                .recv_timeout(Duration::from_secs(30))
18635                .expect("the restart reached the replacement's record write");
18636            let conn = ConnectionInfo {
18637                schema: SCHEMA_VERSION,
18638                wire_version: None,
18639                endpoints: vec![Endpoint {
18640                    host: "127.0.0.1".to_owned(),
18641                    port,
18642                }],
18643                key,
18644                daemon_id,
18645                pid: std::process::id(),
18646                daemon_ver: "test".to_owned(),
18647            };
18648            tokio::runtime::Builder::new_current_thread()
18649                .enable_all()
18650                .build()
18651                .unwrap()
18652                .block_on(async {
18653                    let started = Instant::now();
18654                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18655                        .await
18656                        .unwrap();
18657                    let result =
18658                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18659                    (result, started.elapsed())
18660                })
18661        });
18662
18663        module.restart(Some(0)).await.unwrap();
18664        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18665            .await
18666            .unwrap();
18667        assert!(
18668            auth.is_ok(),
18669            "the daemon must answer a module's handshake while a restart waits on the \
18670             live-children record write; got {auth:?} after {elapsed:?}"
18671        );
18672
18673        tokio::time::timeout(Duration::from_secs(20), async {
18674            while lock_snapshot(&module.inner.snapshot)
18675                .unwrap()
18676                .reported_pid()
18677                .is_none_or(|pid| pid == old_pid)
18678            {
18679                tokio::time::sleep(Duration::from_millis(10)).await;
18680            }
18681        })
18682        .await
18683        .expect("the restart published its replacement");
18684        module.stop().await.unwrap();
18685        server.abort();
18686    }
18687}
18688
18689#[cfg(all(test, unix))]
18690mod retained_status_tests {
18691    use super::*;
18692
18693    #[tokio::test]
18694    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18695        use crate::live_children::{ContainerIdentity, RetainedReason};
18696        for (reason, carried, expected) in [
18697            (
18698                RetainedReason::ContainerSurvivors(vec![4242]),
18699                true,
18700                Some("waiting: container survivors (pids [4242])".to_owned()),
18701            ),
18702            (
18703                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18704                true,
18705                Some(
18706                    "waiting: container survivors could not be verified: member inspection denied"
18707                        .to_owned(),
18708                ),
18709            ),
18710            (RetainedReason::Survived, false, None),
18711        ] {
18712            let supervisor =
18713                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18714            let spec = ModuleSpec {
18715                module_id: "retained-status-fixture".into(),
18716                program: PathBuf::from("/not-spawned"),
18717                args: Vec::new(),
18718                env: Vec::new(),
18719                reserved: false,
18720                reserved_prefixes: Vec::new(),
18721                protocol: ModuleProtocol::None,
18722                overlap: ModuleOverlap::Exclusive,
18723                carry_containment: carried,
18724            };
18725            let module = supervisor
18726                .supervise_configured(spec.clone(), false)
18727                .unwrap();
18728            // No process is launched. Invalid group zero is deliberate: if a
18729            // status read wrongly rechecks, verification changes the reason
18730            // and fails the assertions without signalling any real process.
18731            let guard = supervisor
18732                .child_roster
18733                .admit(
18734                    spec.module_id,
18735                    0,
18736                    ModuleProtocol::None,
18737                    None,
18738                    crate::child_roster::RecordedIdentity {
18739                        container: carried.then_some(ContainerIdentity {
18740                            pgid: Some(0),
18741                            created_start_time: Some(0),
18742                            cgroup_path: None,
18743                            ..ContainerIdentity::test_metadata()
18744                        }),
18745                        ..crate::child_roster::RecordedIdentity::default()
18746                    },
18747                )
18748                .unwrap();
18749            guard.retain(reason);
18750            let status = module.status().unwrap();
18751            assert_eq!(status.waiting_for_previous_detail, expected.clone());
18752            assert_eq!(status.health.detail, expected);
18753            assert!(!status.process_alive);
18754        }
18755    }
18756}