Skip to main content

subc_daemon/
supervise.rs

1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5    collections::{HashMap, HashSet, VecDeque},
6    error::Error,
7    fmt, io,
8    path::PathBuf,
9    process::{ExitStatus, Stdio},
10    sync::{Arc, Mutex, OnceLock},
11    time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22    manifest::{SelfSignalKind, SignalAnchor},
23    session::{
24        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25        MODULE_CONTROL_OP_HEALTH_CHECK,
26    },
27    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30    process::{Child, Command},
31    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32    task::JoinHandle,
33    time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38    child_roster::ChildRoster,
39    daemon_config::{
40        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41    },
42    forwarding::{
43        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44        ModuleDrainTarget, PendingModuleControlRpc,
45    },
46    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47    registry::{ConnectionId, RegistrationEndReason, RegistryError},
48    stderr_tail::{
49        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50        StderrTailSnapshot,
51    },
52    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53    Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75
76#[cfg(all(test, unix))]
77mod slow_retry_tests;
78
79/// Command-line flag used by supervised modules to find subc.
80///
81/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
82/// The path points at the TCP+key connection file; it is not an ambient signal and
83/// is never inherited by standalone children.
84pub const SUBC_ARG: &str = "--subc";
85
86const DEFAULT_MAX_RESTARTS: u32 = 3;
87const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
88const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
89/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
90/// to contain a real crash loop (which respawns in seconds) and short enough
91/// that unrelated crashes hours apart never accumulate into a permanent stop.
92const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
93/// How long a drain waits for already-dispatched requests to finalize before
94/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
95/// not RPC-scale: the original 2s value silently cut nearly every real tool
96/// call at the fence, making the wait-for-finalize design decorative for the
97/// workloads it existed for. Quiescence short-circuits, so an idle module
98/// restarts immediately regardless of this value; the budget is spent only
99/// when a genuine in-flight request is worth finishing. Per-module override:
100/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
101/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
102/// where a stuck request will never settle).
103pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
104pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
105const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
106const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
107/// How long a drain waits after one busy-gauge probe completes before it asks
108/// again.
109///
110/// A drain re-probes on this short interval of its own rather than on the
111/// module's health cadence. The default cadence (30 s) equals the default
112/// drain ceiling, so a cadence-paced drain that found the module busy at its
113/// first probe never asked a second time: it waited out the whole ceiling and
114/// forced GOODBYE even after the gauges had read zero. The interval is counted
115/// from the moment the previous probe finished, and probes run one at a time,
116/// so a slow module is never asked more often than it can answer. Periodic
117/// health checks outside a drain keep using the configured cadence.
118const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
119/// How long a restart waits for the exited process's output readers.
120///
121/// The restart does not depend on either output reader finishing. A stderr
122/// reader still running at this bound is left running, and its later output goes
123/// into the exited process's own section of the stderr ring (see
124/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
125/// bound no longer decides whether a crash's last lines are kept: under load
126/// the reader may simply not have been scheduled yet, and cutting it there
127/// discarded exactly the lines that explained the crash.
128///
129/// What the bound still decides is when the tail starts reporting
130/// `Incomplete`: a pipe open past it usually means a descendant of the exited
131/// process still holds it, and the tail cannot claim to be whole until that
132/// pipe closes. That is also the only case in which the wait costs the restart
133/// anything, because a pipe with no other holder reaches EOF when the process
134/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
135/// `Captured` at EOF with nothing lost.
136///
137/// The stdout reader carries no ring, only the capture file. It also continues
138/// to EOF after this bound: child exit closes the writer but does not mean the
139/// reader has consumed all buffered bytes. Late output from either pipe can
140/// therefore reach the shared capture file after the successor starts.
141const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
142/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
143pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
144const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
145/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
146/// dropped because the subscriber stopped draining its frames. The detail's
147/// `first_undelivered_cursor` names the first event it did not receive; the
148/// client resubscribes from the last cursor it did receive.
149pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
150
151#[cfg(target_os = "macos")]
152struct PrivacyExec {
153    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
154    deadline: tokio::time::Instant,
155    expected: Option<subc_os::FileIdentity>,
156    trampoline: Option<subc_os::FileIdentity>,
157    script: bool,
158    module_id: String,
159    #[cfg(test)]
160    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
161}
162
163#[cfg(all(test, target_os = "macos"))]
164pub(crate) fn test_privacy_trampoline() -> PathBuf {
165    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
166    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
167    let path = std::env::current_exe()
168        .unwrap()
169        .parent()
170        .unwrap()
171        .parent()
172        .unwrap()
173        .join("privacy-trampoline-fixture");
174    // Without the fixture every macOS spawn is refused, and the tests that
175    // spawn fail later as a module in state Failed, which names the wrong
176    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
177    assert!(
178        path.exists(),
179        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
180         --bins --features test-support` or `cargo test -p subc-daemon` first",
181        path.display()
182    );
183    path
184}
185
186#[cfg(target_os = "macos")]
187fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
188    use std::io::Read;
189    let mut probe = std::process::Command::new(path)
190        .args(["__disclaim-exec", "--probe"])
191        .stdin(Stdio::null())
192        .stdout(Stdio::piped())
193        .stderr(Stdio::piped())
194        .spawn()
195        .map_err(|error| {
196            format!(
197                "privacy trampoline probe failed for {}: {error}",
198                path.display()
199            )
200        })?;
201    let deadline = std::time::Instant::now() + Duration::from_secs(5);
202    let status = loop {
203        match probe.try_wait() {
204            Ok(Some(status)) => break status,
205            Ok(None) if std::time::Instant::now() < deadline => {
206                std::thread::sleep(Duration::from_millis(5))
207            }
208            result => {
209                let _ = probe.kill();
210                let _ = probe.wait();
211                return Err(format!(
212                    "privacy trampoline probe failed or timed out for {}: {result:?}",
213                    path.display()
214                ));
215            }
216        }
217    };
218    let mut answer = String::new();
219    if let Some(stdout) = probe.stdout.take() {
220        let _ = stdout.take(256).read_to_string(&mut answer);
221    }
222    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
223        return Ok(());
224    }
225    let mut diagnostic = String::new();
226    if let Some(stderr) = probe.stderr.take() {
227        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
228    }
229    let cause = diagnostic
230        .trim()
231        .strip_prefix("ck-subc: own privacy identity refused: ")
232        .unwrap_or("binary does not implement the privacy trampoline protocol");
233    Err(format!(
234        "{cause}: probe of {} exited {status}",
235        path.display()
236    ))
237}
238
239#[cfg(target_os = "macos")]
240fn privacy_command(
241    spec: &ModuleSpec,
242    roster: &ChildRoster,
243) -> Result<
244    (
245        Command,
246        Option<PrivacyExec>,
247        subc_os::privacy_identity::ExecAcknowledgement,
248    ),
249    SuperviseError,
250> {
251    let failure = |cause: String| {
252        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
253        SuperviseError::Spawn {
254            program: spec.program.clone(),
255            source: io::Error::other(cause),
256            cgroup_path: None,
257        }
258    };
259    let trampoline = roster.privacy_trampoline().map_err(failure)?;
260    // Resolve PATH with the same environment the Command will receive. For
261    // scripts retain the existing orphan-identity rule: the kernel chooses
262    // the interpreter, and its observed image is the one recorded. Do not
263    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
264    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
265        let path = spec
266            .env
267            .iter()
268            .find(|(key, _)| key == "PATH")
269            .map(|(_, value)| std::ffi::OsString::from(value))
270            .or_else(|| std::env::var_os("PATH"))
271            .unwrap_or_else(|| "/usr/bin:/bin".into());
272        std::env::split_paths(&path)
273            .map(|dir| dir.join(&spec.program))
274            .find(|path| path.is_file())
275            .unwrap_or_else(|| spec.program.clone())
276    } else {
277        spec.program.clone()
278    };
279    let expected = subc_os::file_identity(&program);
280    let trampoline_image = subc_os::file_identity(&trampoline);
281    let script = {
282        use std::io::Read;
283        let mut prefix = [0u8; 2];
284        std::fs::File::open(&program)
285            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
286    };
287    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
288        return Err(failure(
289            "privacy identity module executable is missing or is the trampoline itself".to_string(),
290        ));
291    }
292    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
293        .map_err(|error| failure(error.to_string()))?;
294    let reader =
295        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
296    let mut command = Command::new(&trampoline);
297    command
298        .arg("__disclaim-exec")
299        .arg(ack.fd().to_string())
300        .arg(&program);
301    Ok((
302        command,
303        Some(PrivacyExec {
304            reader,
305            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
306            expected,
307            trampoline: trampoline_image,
308            script,
309            module_id: spec.module_id.clone(),
310            #[cfg(test)]
311            confirmation_pause: spec
312                .env
313                .iter()
314                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
315                .map(|(_, address)| (address.clone(), None)),
316        }),
317        ack,
318    ))
319}
320
321struct SupervisedChild {
322    subtree_target: crate::subtree_resources::Target,
323    child: containment::ProcessChild,
324    exec: crate::activity::ExecOwner,
325    #[cfg(windows)]
326    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
327    #[cfg(target_os = "macos")]
328    privacy_exec: Option<PrivacyExec>,
329    /// Set once this launch's exec acknowledgement confirms the module image.
330    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
331    /// `subc_os::privacy_identity`), which then replaces itself with the
332    /// module. The supervisor owns and can kill that pid from spawn, but
333    /// status readers report it only after this latch is set, so nothing
334    /// reports the trampoline's image as the module's.
335    #[cfg(target_os = "macos")]
336    report_ready: Arc<OnceLock<()>>,
337    /// Supervisor failure detail retained for the terminal record, including
338    /// image refusals and first-HELLO timeouts after a successful spawn.
339    spawn_failure: Option<String>,
340    /// The protocol this process was launched with. A reload can store a new
341    /// launch spec with a different protocol, but that takes effect only at the
342    /// next spawn, so this process keeps being handled by the protocol it
343    /// actually speaks.
344    protocol: ModuleProtocol,
345    /// This container's cgroup name: a bounded module/slot label followed by a
346    /// unique suffix (when cgroup placement is on). Carried replacements reuse
347    /// the name; ordinary launches each get a fresh container. A
348    /// retired process in a slot may still be draining when a later one is
349    /// spawned into that slot, so the suffix keeps the later process out of
350    /// the retired one's cgroup, which is the domain a kill applies to.
351    #[cfg(target_os = "linux")]
352    module_id: String,
353    #[cfg(target_os = "linux")]
354    cgroup_placement: Option<subc_cgroup::Placement>,
355    /// The job for an ordinary, non-carried launch. For carried
356    /// launches the actor owns it and the child holds only a weak container link.
357    ///
358    /// Dropping this handle is what reaps a surviving tree when no supervisor
359    /// code runs — a daemon crash — because the job carries
360    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
361    ///
362    /// That limit is not crash-only, and the difference is worth knowing: a
363    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
364    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
365    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
366    /// module at once. Before this change they survived that, saw EOF on the
367    /// control socket, and ran their own teardown; Unix keeps that path
368    /// deliberately, so a module can seal a WAL or close a capture rather than
369    /// be killed mid-write. So this trades graceful teardown on every Windows
370    /// daemon stop for containment on a crash, which is the right way round
371    /// today: orphaned GPU workers are a reported, recurring problem, and the
372    /// modules that write most heavily do not run on Windows.
373    ///
374    /// The fix is a real Windows stop path — the daemon draining before it
375    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
376    /// reaches only what the drain left behind, which is what it should reach.
377    #[cfg(windows)]
378    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
379    stdout_pump: Option<JoinHandle<()>>,
380    stderr_pump: Option<StderrPump>,
381    stderr_ring: Arc<Mutex<StderrRing>>,
382    /// Conservative monotonic bound sampled before `command.spawn()`, not when
383    /// identity recording finishes or the spawn generation is published.
384    spawned_at: Instant,
385    /// Ordinary spawns must send their first HELLO within this budget. Once
386    /// accepted, a later disconnect must not restart the registration clock.
387    registration_deadline: Option<Instant>,
388    registration_watch: Option<RegistrationWatch>,
389    spawned_at_ms: u64,
390    spawned_from: PathBuf,
391    launch_spec_digest: LaunchSpecDigest,
392    spawned_file_identity: Option<SpawnedFileIdentity>,
393    process_start_time: Option<u64>,
394    process_identity: Option<ProcessIdentity>,
395    pid: u32,
396    /// This process's entry in the daemon's child roster, released when the
397    /// process is reaped or this handle is dropped.
398    roster_guard: Option<crate::child_roster::RosterGuard>,
399}
400
401impl SupervisedChild {
402    fn registered_since_spawn(&self) -> bool {
403        self.registration_watch
404            .as_ref()
405            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
406    }
407
408    fn id(&self) -> Option<u32> {
409        Some(self.pid)
410    }
411
412    fn process_identity(&self) -> Option<ProcessIdentity> {
413        self.process_identity
414    }
415
416    async fn wait(&mut self) -> io::Result<ExitStatus> {
417        #[cfg(target_os = "macos")]
418        self.confirm_privacy_exec().await;
419        // The roster entry is NOT released here. A daemon shutdown waits for the
420        // roster to empty and then exits the process, so releasing at the reap
421        // let it exit before the exit handler wrote this child's terminal record
422        // (the stderr drain and snapshot update sit in between), and the
423        // shutdown's own `daemon_shutdown` record was intermittently lost. The
424        // caller releases it after recording the exit (`release_roster`), and
425        // dropping the handle releases it too.
426        let result = self.child.wait().await;
427        #[cfg(target_os = "linux")]
428        if result.is_ok() && self.child.container().is_none() {
429            if let Some(placement) = self.cgroup_placement.as_ref() {
430                // Ordinary children still own a per-spawn cgroup: returning
431                // from wait must leave it empty and removed, as before carry.
432                // Keep cleanup ownership across cancellation until it finishes.
433                cleanup_reaped_cgroup(placement, &self.module_id).await;
434                self.cgroup_placement = None;
435            }
436        }
437        result
438    }
439
440    async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
441        if self.child.container().is_some() {
442            if !carry {
443                self.child.terminate_container().await;
444            }
445        } else {
446            #[cfg(target_os = "linux")]
447            if let Some(placement) = self.cgroup_placement.as_ref() {
448                cleanup_reaped_cgroup(placement, &self.module_id).await;
449                self.cgroup_placement = None;
450            }
451        }
452        self.child.reap().await?;
453        self.release_roster();
454        Ok(())
455    }
456
457    #[cfg(target_os = "macos")]
458    async fn confirm_privacy_exec(&mut self) {
459        let Some(pending) = &mut self.privacy_exec else {
460            return;
461        };
462        let result = tokio::time::timeout_at(pending.deadline, async {
463            let mut record = Vec::new();
464            loop {
465                let mut ready = pending.reader.readable().await?;
466                let read = ready.try_io(|reader| {
467                    use std::io::Read;
468                    let mut reader = reader.get_ref();
469                    let mut buffer = [0u8; 256];
470                    reader.read(&mut buffer).map(|count| (count, buffer))
471                });
472                match read {
473                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
474                    Ok(Ok((count, buffer))) => {
475                        if record.len() + count > 1024 {
476                            return Err(io::Error::other(
477                                "privacy exec refusal record is too long",
478                            ));
479                        }
480                        record.extend_from_slice(&buffer[..count]);
481                    }
482                    Ok(Err(error)) => return Err(error),
483                    Err(_) => continue,
484                }
485            }
486        })
487        .await;
488        #[cfg(test)]
489        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
490            use tokio::io::{AsyncReadExt, AsyncWriteExt};
491            if stream.is_none() {
492                *stream = Some(
493                    tokio::net::TcpStream::connect(address.as_str())
494                        .await
495                        .unwrap(),
496                );
497                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
498                stream
499                    .as_mut()
500                    .unwrap()
501                    .write_all(&self.pid.to_ne_bytes())
502                    .await
503                    .unwrap();
504            }
505            let mut reply = [0];
506            stream
507                .as_mut()
508                .unwrap()
509                .read_exact(&mut reply)
510                .await
511                .unwrap();
512            pending.confirmation_pause = None;
513            if &reply == b"F" {
514                Ok(Err(io::Error::other("injected late exec refusal")))
515            } else {
516                assert_eq!(&reply, b"X");
517                result
518            }
519        } else {
520            result
521        };
522        // Keep the reader in self across await: select cancellation must not
523        // discard the handshake or reset its original five-second deadline.
524        let pending = self.privacy_exec.as_ref().expect("pending exec");
525        let cause = match result {
526            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
527            Ok(Err(error)) => Some(format!(
528                "privacy identity exec acknowledgement failed: {error}"
529            )),
530            Ok(Ok(record)) if !record.is_empty() => Some(
531                std::str::from_utf8(&record)
532                    .ok()
533                    .and_then(|record| {
534                        record
535                            .trim()
536                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
537                    })
538                    .filter(|cause| !cause.is_empty())
539                    .unwrap_or("invalid privacy exec refusal record")
540                    .to_string(),
541            ),
542            Ok(Ok(_)) => match self.child.try_wait() {
543                // Empty EOF is the exec acknowledgement. A real module may exit
544                // immediately, including with a reserved trampoline status; no
545                // image is admitted, and its ordinary exit contract stays intact.
546                Ok(Some(_status)) => None,
547                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
548                Ok(None) => {
549                    let image = observe_spawned_image(self.pid);
550                    if let Some(image) = image.filter(|image| {
551                        image.executable.is_some()
552                            && image.executable != pending.trampoline
553                            && (image.executable == pending.expected || pending.script)
554                    }) {
555                        if let Err(error) = self
556                            .roster_guard
557                            .as_ref()
558                            .map(|guard| guard.confirm_executable(image))
559                            .transpose()
560                        {
561                            Some(format!(
562                                "could not durably record confirmed module image: {error}"
563                            ))
564                        } else {
565                            let _ = self.report_ready.set(());
566                            info!(module_id = %pending.module_id, pid = self.pid,
567                                "module spawned with own privacy identity (responsibility disclaimed)");
568                            None
569                        }
570                    } else if image.is_none()
571                        || image.is_some_and(|image| image.executable.is_none())
572                    {
573                        // A process can exit between try_wait and the kernel
574                        // image read. Empty EOF already acknowledged exec, so
575                        // preserve that module's ordinary exit rather than
576                        // mislabel a disappearing image as trampoline refusal.
577                        // Keep pending in self across await so cancellation does
578                        // not discard validation or reset its original deadline.
579                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
580                            Ok(Ok(_status)) => None,
581                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
582                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
583                        }
584                    } else {
585                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
586                    }
587                }
588            },
589        };
590        let pending = self.privacy_exec.take().expect("pending exec");
591        // Exec confirmation describes launch success, not roster retirement.
592        // Keep even a reaped child recorded until its terminal history exists.
593        self.exec.0.finish(if cause.is_some() {
594            crate::activity::ExecState::Refused
595        } else {
596            crate::activity::ExecState::CommittedExec
597        });
598        if let Some(cause) = cause {
599            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
600            self.spawn_failure = Some(cause);
601            // No image is admitted on failure. Reach the entire fresh process
602            // group, including a module which spawned a helper before refusal.
603            if let Some(container) = self.child.container() {
604                container.hard_kill();
605            } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
606                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
607            }
608            let _ = self.child.start_kill();
609        }
610    }
611
612    /// Releases the leader's roster entry after recording and finalizing its
613    /// exit. A carried container retains its separate roster entry and, for a
614    /// Unix process group, its unreaped first leader that reserves the group ID.
615    fn release_roster(&mut self) {
616        self.roster_guard = None;
617    }
618
619    /// Kill the child and, where containment is available, its process tree.
620    ///
621    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
622    /// helper process leaked the helper — the Synapse embedding module's CUDA
623    /// worker holds the GPU allocation, so the leak cost VRAM until the next
624    /// restart of something else. Terminating the job reaches grandchildren that
625    /// a tree walk cannot, including one whose parent has already exited and
626    /// been reparented away.
627    ///
628    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
629    /// direct-child kill still decides the outcome, so containment can never
630    /// change whether a module is reported as stopped.
631    fn start_kill(&mut self) -> io::Result<()> {
632        if let Some(container) = self.child.container() {
633            container.hard_kill();
634            return self.kill_leader();
635        }
636        #[cfg(windows)]
637        if let Some(job) = &self.job {
638            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
639                debug!(
640                    error = %error,
641                    "job termination failed; the direct-child kill still owns the outcome"
642                );
643            }
644        }
645        #[cfg(target_os = "linux")]
646        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
647        self.child.start_kill()
648    }
649
650    fn kill_leader(&mut self) -> io::Result<()> {
651        self.child.start_kill()
652    }
653
654    async fn drain_stderr(&mut self, module_id: &str) {
655        if let Some(pump) = self.stdout_pump.take() {
656            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
657        }
658
659        let Some(pump) = self.stderr_pump.take() else {
660            return;
661        };
662        settle_stderr_pump(
663            module_id,
664            &self.stderr_ring,
665            pump,
666            STDERR_PUMP_DRAIN_TIMEOUT,
667        )
668        .await;
669    }
670}
671
672/// Bound the restart's wait, not the lifetime of the exited child's stdout
673/// reader. Slow pipe reads or capture writes must not turn the bound into lost
674/// output; dropping the handle lets the reader finish at EOF, like stderr.
675async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
676    match timeout(bound, &mut pump).await {
677        Ok(Ok(())) => {}
678        Ok(Err(error)) => {
679            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
680        }
681        Err(_) => {
682            drop(pump);
683            warn!(
684                module_id,
685                waited = ?bound,
686                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
687            );
688        }
689    }
690}
691
692/// The reader task for one process's stderr, with the ring generation its
693/// lines are attributed to.
694struct StderrPump {
695    task: JoinHandle<()>,
696    generation: u64,
697}
698
699/// Retire an exited process's stderr reader and wait up to `bound` for it to
700/// reach EOF. A reader still running at the bound is detached, not stopped: it
701/// keeps filling the exited process's section of the ring until its pipe
702/// closes, and the tail reads `Incomplete` until then. See
703/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
704async fn settle_stderr_pump(
705    module_id: &str,
706    ring: &Arc<Mutex<StderrRing>>,
707    pump: StderrPump,
708    bound: Duration,
709) {
710    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
711    let StderrPump {
712        mut task,
713        generation,
714    } = pump;
715    lock().retire_pump(generation);
716    match timeout(bound, &mut task).await {
717        Ok(Ok(())) => {}
718        Ok(Err(err)) => {
719            let mut ring = lock();
720            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
721            ring.finish_pump(generation);
722            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
723        }
724        Err(_) => {
725            // Dropping the handle detaches the task; it ends at EOF on its pipe.
726            drop(task);
727            lock().mark_pump_late(
728                generation,
729                format!(
730                    "stderr of the exited process had not reached EOF {bound:?} after it was \
731                     retired (a descendant may still hold the pipe open); lines it still \
732                     writes are kept in that process's section"
733                ),
734            );
735            warn!(
736                module_id,
737                waited = ?bound,
738                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
739            );
740        }
741    }
742}
743
744fn registration_release_events() -> &'static watch::Sender<u64> {
745    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
746    EVENTS.get_or_init(|| {
747        let (sender, _receiver) = watch::channel(0);
748        sender
749    })
750}
751
752pub(crate) fn notify_registration_release() {
753    let events = registration_release_events();
754    let next_generation = (*events.borrow()).wrapping_add(1);
755    events.send_replace(next_generation);
756}
757
758/// How to launch one singleton module process.
759#[derive(Debug, Clone, PartialEq, Eq)]
760pub struct ModuleSpec {
761    pub module_id: String,
762    pub program: PathBuf,
763    pub args: Vec<String>,
764    pub env: Vec<(String, String)>,
765    /// When true this is a reserved module: each spawn gets a fresh one-time launch
766    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
767    /// process can register this module_id (a security-boundary module like the
768    /// credential vault must not be impersonable while it is down/restarting).
769    pub reserved: bool,
770    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
771    /// Prefixes come from daemon config and must end in `:` before they reach the
772    /// supervisor; the owner module's current spawn nonce authorizes claims under
773    /// each prefix.
774    pub reserved_prefixes: Vec<String>,
775    /// The wire protocol this module speaks, as DECLARED in daemon config.
776    ///
777    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
778    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
779    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
780    /// and NO launch nonce, and a clean exit the daemon did not request is
781    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
782    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
783    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
784    /// because a process ignores an environment variable it does not read.
785    ///
786    /// The argument is the part that cannot be "harmless to a process that
787    /// ignores it": a stock binary exits on an unknown flag before it listens
788    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
789    /// first conformance run against this mode found it. The nonce is withheld
790    /// because a process that will never present it gains nothing from holding
791    /// it, and a secret in the environment of a process that does not need it is
792    /// a leak surface for no benefit.
793    pub protocol: ModuleProtocol,
794    /// Whether two processes of this module may run at once, which is what a
795    /// blue/green swap does for the length of its overlap. Declared in daemon
796    /// config because the daemon must be able to answer it while the module is
797    /// down, and so a module cannot talk itself into it after registering.
798    pub overlap: ModuleOverlap,
799    /// Keep module-started helpers in the same container across replacements.
800    /// This is a daemon-config policy, fixed for the lifetime of a container;
801    /// stopping the module or losing its supervisor still destroys the container.
802    pub carry_containment: bool,
803}
804
805/// Only the configured launch inputs enter this value; spawned connection paths,
806/// nonces and inherited environment never do. Its bytes remain daemon-private.
807#[derive(Clone, PartialEq, Eq)]
808pub(crate) struct LaunchSpecDigest([u8; 32]);
809
810impl fmt::Debug for LaunchSpecDigest {
811    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
812        f.write_str("LaunchSpecDigest([redacted])")
813    }
814}
815
816impl LaunchSpecDigest {
817    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
818        fn field(hash: &mut Sha256, bytes: &[u8]) {
819            hash.update((bytes.len() as u64).to_le_bytes());
820            hash.update(bytes);
821        }
822        let mut hash = Sha256::new();
823        if let Some(program) = spec.program.to_str() {
824            field(&mut hash, b"utf8-program");
825            field(&mut hash, program.as_bytes());
826        } else {
827            // The daemon config is UTF-8, but in-process callers may provide
828            // native paths: lossy conversion would collapse distinct programs.
829            #[cfg(unix)]
830            {
831                use std::os::unix::ffi::OsStrExt;
832                field(&mut hash, b"unix-program");
833                field(&mut hash, spec.program.as_os_str().as_bytes());
834            }
835            #[cfg(windows)]
836            {
837                use std::os::windows::ffi::OsStrExt;
838                field(&mut hash, b"windows-program");
839                let bytes = spec
840                    .program
841                    .as_os_str()
842                    .encode_wide()
843                    .flat_map(u16::to_le_bytes)
844                    .collect::<Vec<_>>();
845                field(&mut hash, &bytes);
846            }
847            #[cfg(not(any(unix, windows)))]
848            field(&mut hash, spec.program.to_string_lossy().as_bytes());
849        }
850        hash.update((spec.args.len() as u64).to_le_bytes());
851        for arg in &spec.args {
852            field(&mut hash, arg.as_bytes());
853        }
854        let mut env = spec.env.iter().collect::<Vec<_>>();
855        env.sort_unstable();
856        hash.update((env.len() as u64).to_le_bytes());
857        for (key, value) in env {
858            field(&mut hash, key.as_bytes());
859            field(&mut hash, value.as_bytes());
860        }
861        field(
862            &mut hash,
863            match spec.protocol {
864                ModuleProtocol::Subc => b"subc",
865                ModuleProtocol::None => b"none",
866            },
867        );
868        Self(hash.finalize().into())
869    }
870}
871
872/// Whether a module tolerates a second process of itself running alongside.
873///
874/// Most modules are single-writer on their store (a WAL, a capture log, a
875/// resident index behind a writer barrier), and two processes on one store
876/// corrupt it. So a swap, which overlaps the old and new process by design,
877/// is refused unless the module's config opts in with `overlap: "safe"`.
878#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
879pub enum ModuleOverlap {
880    /// Never run two processes of this module at once. The default.
881    #[default]
882    Exclusive,
883    /// The module has said a second process of itself is harmless for the
884    /// length of a swap.
885    ///
886    /// Declare it only if a second instance can run for a few seconds without
887    /// touching ANY single-writer store: every database, WAL, index, projector
888    /// and scheduled job the module owns. A lease on part of that state is not
889    /// enough. broca's session lease guards WAL appends while its run index, its
890    /// store projector and its archive fold timer (which unlinks live WAL files)
891    /// stay single-writer, so broca is exclusive despite holding a lease. The
892    /// refusal only fires after this has been decided, so the decision is the
893    /// check.
894    Safe,
895}
896
897impl ModuleOverlap {
898    pub fn as_str(self) -> &'static str {
899        match self {
900            Self::Exclusive => "exclusive",
901            Self::Safe => "safe",
902        }
903    }
904}
905
906/// Environment variable telling a spawned module which case it was started
907/// for, before it sends HELLO. Only a swap candidate carries it, as
908/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
909///
910/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
911/// longer because nobody waits on it, while a plain restart must flip ready
912/// quickly because callers see `module_warming` until it does. Absence means
913/// plain restart, the safe reading. The daemon trusts nothing about it; the
914/// candidate is proven by its launch nonce at HELLO.
915pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
916/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
917pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
918/// How long a swap waits for its candidate to register and declare itself
919/// ready when the operator does not say. A module warming as a swap candidate
920/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
921/// daemon allows that plus time to start the process and send HELLO.
922pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
923
924/// Bounded restart policy for crash exits.
925///
926/// `max_restarts` is the number of replacement processes allowed after the
927/// initial spawn WITHIN `window`. After that many crash restarts inside one
928/// window the module enters [`ModuleState::Failed`] and the supervisor stops
929/// the crash loop.
930///
931/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
932/// and that only survived because crashes were rare: a module that crashed
933/// three times across a week was disabled forever by crashes that had nothing
934/// to do with each other. That stopped being survivable once modules began
935/// exiting non-zero whenever the daemon's connection to them drops, because
936/// then every daemon-side connection drop spends a unit of the same budget and
937/// one flappy hour permanently stops a healthy module. Restarts older than
938/// `window` release their slot, so a module that crashed twice yesterday has a
939/// full budget today, while a genuine crash loop -- which is fast by
940/// definition -- still reaches the cap and stops.
941#[derive(Debug, Clone, Copy, PartialEq, Eq)]
942pub struct RestartPolicy {
943    pub max_restarts: u32,
944    /// Base delay before a crash replacement. The actual delay escalates with
945    /// the number of recent crash replacements and is capped by `max_backoff`.
946    pub backoff: Duration,
947    /// Maximum delay before a crash replacement.
948    pub max_backoff: Duration,
949    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
950    /// budget effectively infinite (nothing is ever in-window), which is why
951    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
952    pub window: Duration,
953}
954
955impl RestartPolicy {
956    /// A policy with the default crash window. Callers that care about the
957    /// window say so with [`Self::with_window`]; the ones that do not are
958    /// asking for the standard rate limit, not for no limit.
959    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
960        Self {
961            max_restarts,
962            backoff,
963            max_backoff: DEFAULT_MAX_BACKOFF,
964            window: DEFAULT_RESTART_WINDOW,
965        }
966    }
967
968    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
969        self.max_backoff = max_backoff;
970        self
971    }
972
973    pub fn with_window(mut self, window: Duration) -> Self {
974        self.window = window;
975        self
976    }
977
978    /// Calculate the capped exponential delay for the next crash replacement.
979    /// `restart_in_window` is zero for the first replacement after an operator
980    /// action (restart, reload, re-enable) cleared the crash ring, or after all
981    /// older crash replacements have aged out of the window.
982    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
983        if self.backoff.is_zero() || self.max_backoff.is_zero() {
984            return Duration::ZERO;
985        }
986
987        let mut delay = self.backoff;
988        for _ in 0..restart_in_window {
989            if delay >= self.max_backoff {
990                return self.max_backoff;
991            }
992            delay = delay
993                .checked_mul(10)
994                .unwrap_or(self.max_backoff)
995                .min(self.max_backoff);
996        }
997        delay.min(self.max_backoff)
998    }
999
1000    /// The one sentence that explains a budget-exhausted stop, used for both the
1001    /// log line and the terminal record so the two cannot drift. It names the
1002    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
1003    /// exactly what this budget is not.
1004    fn budget_exhausted_detail(&self) -> String {
1005        format!(
1006            "crash budget exhausted: max_restarts={} within window_secs={}",
1007            self.max_restarts,
1008            self.window.as_secs()
1009        )
1010    }
1011}
1012
1013impl Default for RestartPolicy {
1014    fn default() -> Self {
1015        Self {
1016            max_restarts: DEFAULT_MAX_RESTARTS,
1017            backoff: DEFAULT_BACKOFF,
1018            max_backoff: DEFAULT_MAX_BACKOFF,
1019            window: DEFAULT_RESTART_WINDOW,
1020        }
1021    }
1022}
1023
1024#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1025struct CrashRestartSchedule {
1026    restart_in_window: u32,
1027    delay: Duration,
1028}
1029
1030/// Whether the daemon itself will bring this module back after the exit being
1031/// handled: it is enabled AND its in-window crash restarts are below the cap.
1032///
1033/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
1034/// the window are dropped here rather than by a timer, so the count is right
1035/// the moment somebody asks and no bookkeeping runs for idle modules.
1036fn daemon_will_restart(
1037    state: &mut SupervisorSnapshot,
1038    policy: &RestartPolicy,
1039    now: Instant,
1040) -> bool {
1041    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1042}
1043
1044const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1045const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1046const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1047const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1048
1049#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1050pub enum HealthAction {
1051    Report,
1052    Restart,
1053    Alert,
1054}
1055
1056impl fmt::Display for HealthAction {
1057    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1058        f.write_str(match self {
1059            Self::Report => "report",
1060            Self::Restart => "restart",
1061            Self::Alert => "alert",
1062        })
1063    }
1064}
1065
1066#[derive(Debug, Clone, PartialEq, Eq)]
1067pub struct HealthConfig {
1068    /// Optional loopback HTTP endpoint for a managed non-wire process.
1069    /// Changing it applies live on rescan; the process protocol changes only
1070    /// at its next spawn.
1071    pub http: Option<String>,
1072    pub cadence: Duration,
1073    pub deadline: Duration,
1074    pub failure_threshold: u32,
1075    pub on_degraded: HealthAction,
1076    pub on_failing: HealthAction,
1077    pub critical: bool,
1078}
1079
1080impl Default for HealthConfig {
1081    fn default() -> Self {
1082        Self {
1083            http: None,
1084            cadence: DEFAULT_HEALTH_CADENCE,
1085            deadline: DEFAULT_HEALTH_DEADLINE,
1086            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1087            on_degraded: HealthAction::Report,
1088            on_failing: HealthAction::Report,
1089            critical: false,
1090        }
1091    }
1092}
1093
1094/// The supervisor's view of one module's health, relayed to clients over
1095/// channel-0 and rendered by `ck health`.
1096///
1097/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1098/// stated here rather than only at the wire type a consumer reads. A reader can
1099/// look up what `None` means; only a writer can silently change it, and the
1100/// writer has no reason to go looking at a downstream contract before editing.
1101///
1102/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1103/// back to `None` on re-registration precisely so a respawned module does not
1104/// carry its predecessor's timestamp — so an old value and an absent one call for
1105/// opposite readings, and anything that defaulted this to a number would make a
1106/// never-probed module indistinguishable from one probed at the epoch.
1107///
1108/// `detail` and `metrics` are `None` when the module published none on this
1109/// probe, which does not mean it reported nothing wrong — it is also the shape
1110/// when the probe never reached it. `last_probe_ms` is what separates those.
1111#[derive(Debug, Clone, PartialEq)]
1112pub struct ModuleHealthStatus {
1113    pub status: SupervisorHealthStatus,
1114    pub last_probe_ms: Option<u64>,
1115    pub detail: Option<String>,
1116    pub metrics: Option<Value>,
1117    pub consecutive_failures: u32,
1118    /// Number of replies received after a recurring health probe's deadline.
1119    /// Unlike a timeout, every increment proves the module was alive.
1120    pub late_answer_count: u64,
1121    /// End-to-end latency of the newest late reply, measured from probe start.
1122    pub last_late_answer_latency_ms: Option<u64>,
1123    pub last_action: Option<String>,
1124    /// Set together with `last_action`; the pair moves as one, and both being
1125    /// absent means no escalation has ever been taken rather than that the last
1126    /// one succeeded.
1127    pub last_action_ms: Option<u64>,
1128}
1129
1130impl Default for ModuleHealthStatus {
1131    fn default() -> Self {
1132        Self {
1133            status: SupervisorHealthStatus::Unknown,
1134            last_probe_ms: None,
1135            detail: None,
1136            metrics: None,
1137            consecutive_failures: 0,
1138            late_answer_count: 0,
1139            last_late_answer_latency_ms: None,
1140            last_action: None,
1141            last_action_ms: None,
1142        }
1143    }
1144}
1145
1146/// Typed lifecycle state for a supervised module.
1147#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1148pub enum ModuleState {
1149    Starting,
1150    Running,
1151    Unresponsive,
1152    Restarting,
1153    Draining,
1154    Stopped,
1155    Failed,
1156    Disabled,
1157}
1158
1159impl fmt::Display for ModuleState {
1160    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1161        f.write_str(match self {
1162            Self::Starting => "starting",
1163            Self::Running => "running",
1164            Self::Unresponsive => "unresponsive",
1165            Self::Restarting => "restarting",
1166            Self::Draining => "draining",
1167            Self::Stopped => "stopped",
1168            Self::Failed => "failed",
1169            Self::Disabled => "disabled",
1170        })
1171    }
1172}
1173
1174/// Supervisor classification of a child-process exit.
1175#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1176pub enum ExitKind {
1177    Clean,
1178    Crash,
1179    DeliberateSeverance,
1180}
1181
1182impl From<ExitKind> for TerminalExitKind {
1183    fn from(kind: ExitKind) -> Self {
1184        match kind {
1185            ExitKind::Clean => Self::Clean,
1186            ExitKind::Crash => Self::Crash,
1187            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1188        }
1189    }
1190}
1191
1192/// Exact process identity retained when a supervised module registers its
1193/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1194#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1195pub(crate) struct ProcessIdentity {
1196    pub(crate) pid: u32,
1197    pub(crate) start_time: u64,
1198}
1199
1200/// Last observed child exit, if any.
1201#[derive(Debug, Clone, PartialEq, Eq)]
1202pub struct ExitReport {
1203    pub kind: ExitKind,
1204    pub code: Option<i32>,
1205    pub signal: Option<i32>,
1206    pub at_ms: u64,
1207}
1208
1209/// Point-in-time module status answerable by subc without forwarding to the
1210/// module process.
1211#[derive(Debug, Clone, PartialEq)]
1212pub struct ModuleStatus {
1213    pub module_id: String,
1214    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1215    pub waiting_for_previous_pid: Option<u32>,
1216    /// Why the previous instance still prevents a new start.
1217    pub waiting_for_previous_detail: Option<String>,
1218    /// The daemon's Windows per-spawn record: `pipe` after the process it
1219    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1220    /// List/status reports use this record, not the source the module sends
1221    /// in its registration message's manifest.provenance.launch_nonce_source.
1222    pub launch_nonce_source: Option<String>,
1223    pub state: ModuleState,
1224    pub enabled: bool,
1225    pub process_alive: bool,
1226    pub registration_active: bool,
1227    /// The module's declared wire protocol, carried beside `live` because it is
1228    /// what makes `live` readable: the two fields answer one question together.
1229    /// While a process is alive this is its launch declaration, not a later
1230    /// pending-reload edit. When down it is the configured next launch protocol.
1231    pub protocol: ModuleProtocol,
1232    /// Whether the module is serving, under the strongest definition the daemon
1233    /// can assert for its protocol.
1234    ///
1235    /// A subc module must also be REGISTERED: its process being alive says
1236    /// nothing about whether it can take a request. A `protocol: "none"` module
1237    /// never registers, so that term is dropped and this falls back to "enabled,
1238    /// running, and the process the daemon launched is alive" -- which is all
1239    /// the daemon observes about a process that speaks no subc wire. It stays a
1240    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1241    /// rather than printing it bare.
1242    pub live: bool,
1243    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1244    /// restarts have already released their slot, so this count can go down
1245    /// without anybody touching the module.
1246    pub restart_count: u32,
1247    /// Replacement processes spawned over this module's entire supervisor lifetime;
1248    /// unlike `restart_count`, this value is never reset by an operator action
1249    /// and never falls out of a window.
1250    pub lifetime_restarts: u32,
1251    pub spawn_generation: u64,
1252    /// The budget `restart_count` is spent against. Carried alongside the count
1253    /// because the count alone does not say how close the module is to being
1254    /// disabled, and reporting one without the other is what makes an
1255    /// about-to-be-retired module look ordinary.
1256    pub max_restarts: u32,
1257    /// The span `restart_count` is counted over. Carried with the pair above for
1258    /// the same reason they are carried together: "2 of 3" means one thing for a
1259    /// ten-minute window and something else entirely for a lifetime.
1260    pub restart_window: Duration,
1261    /// Effective drain and restart timing policy used by this running module.
1262    /// These values are carried together with the restart budget so status
1263    /// readers can compare configured intent with what the supervisor applied.
1264    pub drain_timeout: Duration,
1265    pub restart_backoff: Duration,
1266    pub restart_max_backoff: Duration,
1267    /// The module's process. On macOS this stays absent while the `ck-subc`
1268    /// launch trampoline is still running in that pid, and appears once the
1269    /// exec acknowledgement confirms the module image has replaced it. Launch
1270    /// time and the supervisor's own hold on the process are unaffected.
1271    pub pid: Option<u32>,
1272    pub spawned_at_ms: Option<u64>,
1273    pub spawned_from: Option<PathBuf>,
1274    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1275    pub process_start_time: Option<u64>,
1276    pub last_exit: Option<ExitReport>,
1277    pub health: ModuleHealthStatus,
1278}
1279
1280#[cfg(test)]
1281#[derive(Debug, Clone, Copy, PartialEq)]
1282struct ActorSelectCheckpoint {
1283    generation: u64,
1284    turn: u64,
1285    registered_connection: Option<ConnectionId>,
1286    next_probe_at: Option<Instant>,
1287    wake_after: Option<Duration>,
1288}
1289
1290#[derive(Debug, Clone, PartialEq)]
1291struct SupervisorSnapshot {
1292    subtree_target: crate::subtree_resources::Target,
1293    failed_retry: FailedRetry,
1294    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1295    #[cfg(windows)]
1296    pipe_consumed: Option<Arc<OnceLock<()>>>,
1297    /// Counts running-child loop turns, not executor polls of a parked wait.
1298    #[cfg(test)]
1299    actor_turns: u64,
1300    /// Running select after exec confirmation and with no unconsumed registry
1301    /// event. Recording a spawn or starting a loop turn is not this barrier.
1302    #[cfg(test)]
1303    actor_select: Option<ActorSelectCheckpoint>,
1304    state: ModuleState,
1305    enabled: bool,
1306    process_alive: bool,
1307    spawned_protocol: Option<ModuleProtocol>,
1308    spawn_failure: Option<String>,
1309    waiting_for_previous_pid: Option<u32>,
1310    waiting_for_previous_detail: Option<String>,
1311    /// When each crash restart was spent, oldest first. This IS the crash
1312    /// budget: its in-window length is the count an operator sees and the count
1313    /// the restart decision is made against, so there is no second counter that
1314    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1315    /// operator actions that used to zero the old lifetime counter.
1316    crash_restarts: VecDeque<Instant>,
1317    lifetime_restarts: u32,
1318    /// Successful child spawns in this daemon incarnation.
1319    ///
1320    /// `lifetime_restarts` was considered and rejected: it starts at zero
1321    /// (line 640), successful initial/operator spawns in `set_running` do not
1322    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1323    /// increments before a successful replacement exists (lines 604, 3846,
1324    /// and 3921), so a failed spawn can consume it. This counter moves only
1325    /// when a live PID is accepted below.
1326    spawn_generation: u64,
1327    pid: Option<u32>,
1328    #[cfg(target_os = "macos")]
1329    report_ready: Option<Arc<OnceLock<()>>>,
1330    /// Last reaped child, retained after current process facts are cleared.
1331    reaped_pid: Option<u32>,
1332    /// Whether a replacement has already been scheduled for the current process.
1333    respawn_pending: bool,
1334    /// A second restart is waiting for the replacement already scheduled.
1335    coalesced_restart_pending: bool,
1336    spawned_at_ms: Option<u64>,
1337    spawned_from: Option<PathBuf>,
1338    launch_spec_digest: Option<LaunchSpecDigest>,
1339    spawned_file_identity: Option<SpawnedFileIdentity>,
1340    process_start_time: Option<u64>,
1341    deliberate_severance: Option<ProcessIdentity>,
1342    last_exit: Option<ExitReport>,
1343    /// Diagnostic attached to the next drain's terminal record, if any.
1344    drain_disposition_detail: Option<String>,
1345    health: ModuleHealthStatus,
1346    /// Whether the current process was started as a swap candidate and so
1347    /// lives in the module's alternate cgroup. The next swap's candidate takes
1348    /// the other one, so the two processes of a swap never share a cgroup. A
1349    /// plain spawn always uses the primary cgroup.
1350    in_alternate_slot: bool,
1351    /// Whether the current `Draining` state ends in a replacement process
1352    /// (restart, reload, health restart) rather than a stop. Only meaningful
1353    /// while `state` is `Draining`; every entry into that state rewrites it.
1354    /// It is what lets route.open answer the retryable `module_reloading` to a
1355    /// consumer that reaches a still-registered process mid-restart, instead of
1356    /// the `supervisor_not_live` a stop or disable deserves.
1357    draining_to_replace: bool,
1358    /// Whether a configuration update has been applied since the current
1359    /// process was spawned, so that process runs an older spec than the one
1360    /// the supervisor now holds. A queued restart is only coalesced into a
1361    /// fresher process when this is false: a restart requested to pick up a
1362    /// new configuration must not be satisfied by a process that predates it.
1363    configuration_updated_since_spawn: bool,
1364}
1365
1366impl SupervisorSnapshot {
1367    /// The pid that status, provenance and resource readings may report. While
1368    /// the launch trampoline still runs in the pid, reading its executable or
1369    /// resource use would describe `ck-subc`, not the module, so none is
1370    /// reported until the exec acknowledgement confirms the module image.
1371    fn reported_pid(&self) -> Option<u32> {
1372        #[cfg(target_os = "macos")]
1373        if self
1374            .report_ready
1375            .as_ref()
1376            .is_some_and(|ready| ready.get().is_none())
1377        {
1378            return None;
1379        }
1380        self.pid
1381    }
1382
1383    fn starting() -> Self {
1384        Self::new(ModuleState::Starting, true)
1385    }
1386
1387    fn disabled() -> Self {
1388        Self::new(ModuleState::Disabled, false)
1389    }
1390
1391    /// Crash restarts still inside `window`, having dropped the ones that are
1392    /// not. Pruning on read is what makes the budget a rate: an instant older
1393    /// than the window stops holding a slot the moment anybody counts.
1394    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1395        while let Some(oldest) = self.crash_restarts.front() {
1396            if now.duration_since(*oldest) > window {
1397                self.crash_restarts.pop_front();
1398            } else {
1399                break;
1400            }
1401        }
1402        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1403    }
1404
1405    /// Spend one unit of the crash budget and record the restart in the ledger.
1406    ///
1407    /// The ring is bounded by the cap because more than `max_restarts` in-window
1408    /// instants can never be reached (the caller refuses the restart first), so
1409    /// anything beyond that is an unbounded queue waiting to happen.
1410    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1411        self.crash_restarts.push_back(now);
1412        while self.crash_restarts.len() > policy.max_restarts as usize {
1413            self.crash_restarts.pop_front();
1414        }
1415        self.lifetime_restarts += 1;
1416    }
1417
1418    /// Reserve one crash-restart slot and calculate the delay before respawning.
1419    /// The count is captured before recording this restart, so the first retry
1420    /// uses the base delay and each later in-window retry escalates once.
1421    fn next_crash_restart(
1422        &mut self,
1423        policy: &RestartPolicy,
1424        now: Instant,
1425    ) -> Option<CrashRestartSchedule> {
1426        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1427        if restart_in_window >= policy.max_restarts {
1428            return None;
1429        }
1430        self.record_crash_restart(policy, now);
1431        Some(CrashRestartSchedule {
1432            restart_in_window,
1433            delay: policy.delay_for_restart(restart_in_window),
1434        })
1435    }
1436
1437    /// Give the module its full budget back, as an operator restart, reload, or
1438    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1439    /// ledger of what actually happened, and an operator action does not unmake
1440    /// the crashes.
1441    fn clear_crash_restarts(&mut self) {
1442        self.crash_restarts.clear();
1443    }
1444
1445    fn new(state: ModuleState, enabled: bool) -> Self {
1446        Self {
1447            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1448            failed_retry: FailedRetry::default(),
1449            exec_outcome: None,
1450            #[cfg(test)]
1451            actor_turns: 0,
1452            #[cfg(test)]
1453            actor_select: None,
1454            state,
1455            enabled,
1456            process_alive: false,
1457            #[cfg(windows)]
1458            pipe_consumed: None,
1459            spawned_protocol: None,
1460            spawn_failure: None,
1461            waiting_for_previous_pid: None,
1462            waiting_for_previous_detail: None,
1463            crash_restarts: VecDeque::new(),
1464            lifetime_restarts: 0,
1465            spawn_generation: 0,
1466            pid: None,
1467            #[cfg(target_os = "macos")]
1468            report_ready: None,
1469            reaped_pid: None,
1470            respawn_pending: false,
1471            coalesced_restart_pending: false,
1472            spawned_at_ms: None,
1473            spawned_from: None,
1474            launch_spec_digest: None,
1475            spawned_file_identity: None,
1476            process_start_time: None,
1477            deliberate_severance: None,
1478            last_exit: None,
1479            drain_disposition_detail: None,
1480            health: ModuleHealthStatus::default(),
1481            in_alternate_slot: false,
1482            draining_to_replace: false,
1483            configuration_updated_since_spawn: false,
1484        }
1485    }
1486}
1487
1488type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1489
1490type SpawnSubscriberKey = (ConnectionId, u64);
1491
1492#[derive(Debug)]
1493struct SpawnSubscriber {
1494    version: u8,
1495    frames: mpsc::Sender<Frame>,
1496    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1497    /// from which event. The full frame channel cannot carry that news, so it
1498    /// travels beside it; see `SpawnEventFeed::subscribe`.
1499    lagged: Option<oneshot::Sender<SpawnCursor>>,
1500}
1501
1502#[derive(Debug)]
1503struct SpawnEventState {
1504    daemon_incarnation: String,
1505    seq: u64,
1506    capacity: usize,
1507    live: HashMap<String, LiveSpawn>,
1508    generations: HashMap<String, u64>,
1509    events: VecDeque<SpawnEvent>,
1510    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1511}
1512
1513impl Default for SpawnEventState {
1514    fn default() -> Self {
1515        Self {
1516            daemon_incarnation: "unconfigured".to_string(),
1517            seq: 0,
1518            capacity: SPAWN_EVENT_RING_CAPACITY,
1519            live: HashMap::new(),
1520            generations: HashMap::new(),
1521            events: VecDeque::new(),
1522            subscribers: HashMap::new(),
1523        }
1524    }
1525}
1526
1527#[derive(Debug, Clone, Default)]
1528struct SpawnEventFeed(
1529    Arc<Mutex<SpawnEventState>>,
1530    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1531);
1532
1533#[derive(Debug, Clone, PartialEq, Eq)]
1534pub(crate) enum SpawnSubscribeRefusal {
1535    ForeignIncarnation { current: String },
1536    TooOld { oldest: SpawnCursor },
1537    Frame(String),
1538}
1539
1540impl SpawnEventFeed {
1541    fn configure_incarnation(&self, daemon_incarnation: String) {
1542        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1543        state.daemon_incarnation = daemon_incarnation;
1544        state.seq = 0;
1545        state.live.clear();
1546        state.generations.clear();
1547        state.events.clear();
1548        state.subscribers.clear();
1549    }
1550
1551    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1552        SpawnCursor {
1553            daemon_incarnation: state.daemon_incarnation.clone(),
1554            seq: state.seq,
1555        }
1556    }
1557
1558    fn snapshot(&self) -> SpawnSnapshot {
1559        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1560        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1561        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1562        SpawnSnapshot {
1563            cursor: Self::cursor(&state),
1564            ring_bound: state.capacity as u64,
1565            live,
1566        }
1567    }
1568
1569    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1570        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1571        let generation = state
1572            .generations
1573            .get(module_id)
1574            .copied()
1575            .unwrap_or(0)
1576            .checked_add(1)
1577            .expect("spawn generation exhausted");
1578        state.generations.insert(module_id.to_string(), generation);
1579        let live = LiveSpawn {
1580            module_id: module_id.to_string(),
1581            spawn_generation: generation,
1582            pid,
1583            spawned_at_ms,
1584        };
1585        state.live.insert(module_id.to_string(), live);
1586        Self::emit_locked(
1587            &mut state,
1588            SpawnEventKind::Spawned,
1589            module_id.to_string(),
1590            generation,
1591            pid,
1592            None,
1593            None,
1594        );
1595        generation
1596    }
1597
1598    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1599        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1600        let Some(live) = state.live.remove(module_id) else {
1601            warn!(
1602                module_id,
1603                "terminal record had no live spawn event identity"
1604            );
1605            return;
1606        };
1607        Self::emit_locked(
1608            &mut state,
1609            SpawnEventKind::Exited,
1610            module_id.to_string(),
1611            live.spawn_generation,
1612            live.pid,
1613            exit_code,
1614            exit_signal,
1615        );
1616    }
1617
1618    /// Report the exit of a process that a swap has already replaced.
1619    ///
1620    /// `emit_exited` removes the module's live entry, which after a swap's
1621    /// cutover describes the promoted candidate, not the old process now
1622    /// exiting. This emits the old generation's exit and leaves the live entry
1623    /// alone unless it still names that generation.
1624    fn emit_superseded_exited(
1625        &self,
1626        module_id: &str,
1627        spawn_generation: u64,
1628        pid: u32,
1629        exit_code: Option<i32>,
1630        exit_signal: Option<i32>,
1631    ) {
1632        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1633        if state
1634            .live
1635            .get(module_id)
1636            .is_some_and(|live| live.spawn_generation == spawn_generation)
1637        {
1638            state.live.remove(module_id);
1639        }
1640        Self::emit_locked(
1641            &mut state,
1642            SpawnEventKind::Exited,
1643            module_id.to_string(),
1644            spawn_generation,
1645            pid,
1646            exit_code,
1647            exit_signal,
1648        );
1649    }
1650
1651    #[allow(clippy::too_many_arguments)]
1652    fn emit_locked(
1653        state: &mut SpawnEventState,
1654        kind: SpawnEventKind,
1655        module_id: String,
1656        spawn_generation: u64,
1657        pid: u32,
1658        exit_code: Option<i32>,
1659        exit_signal: Option<i32>,
1660    ) {
1661        state.seq = state
1662            .seq
1663            .checked_add(1)
1664            .expect("spawn event sequence exhausted");
1665        let event = SpawnEvent {
1666            cursor: Self::cursor(state),
1667            kind,
1668            module_id,
1669            spawn_generation,
1670            pid,
1671            exit_code,
1672            exit_signal,
1673        };
1674        state.events.push_back(event.clone());
1675        while state.events.len() > state.capacity {
1676            state.events.pop_front();
1677        }
1678        let body = match serde_json::to_vec(&event) {
1679            Ok(body) => body,
1680            Err(error) => {
1681                error!(%error, "failed to serialize supervisor spawn event");
1682                return;
1683            }
1684        };
1685        state.subscribers.retain(|(connection_id, corr), subscriber| {
1686            let frame = Frame::build_with_version(
1687                subscriber.version,
1688                FrameType::StreamData,
1689                control_flags(),
1690                0,
1691                0,
1692                *corr,
1693                body.clone(),
1694            );
1695            match frame {
1696                Ok(frame) => {
1697                    if subscriber.frames.try_send(frame).is_ok() {
1698                        true
1699                    } else {
1700                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1701                        if let Some(lagged) = subscriber.lagged.take() {
1702                            let _ = lagged.send(event.cursor.clone());
1703                        }
1704                        false
1705                    }
1706                }
1707                Err(error) => {
1708                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1709                    false
1710                }
1711            }
1712        });
1713    }
1714
1715    fn subscribe(
1716        &self,
1717        connection_id: ConnectionId,
1718        corr: u64,
1719        version: u8,
1720        since: Option<SpawnCursor>,
1721        sink: FrameSink,
1722    ) -> Result<(), SpawnSubscribeRefusal> {
1723        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1724        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1725        {
1726            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1727            let replay = if let Some(since) = since {
1728                if since.daemon_incarnation != state.daemon_incarnation {
1729                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1730                        current: state.daemon_incarnation.clone(),
1731                    });
1732                }
1733                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1734                    if since.seq < oldest.seq.saturating_sub(1) {
1735                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1736                    }
1737                }
1738                state
1739                    .events
1740                    .iter()
1741                    .filter(|event| event.cursor.seq > since.seq)
1742                    .cloned()
1743                    .collect::<Vec<_>>()
1744            } else {
1745                Vec::new()
1746            };
1747            for event in replay {
1748                let body = serde_json::to_vec(&event)
1749                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1750                let frame = Frame::build_with_version(
1751                    version,
1752                    FrameType::StreamData,
1753                    control_flags(),
1754                    0,
1755                    0,
1756                    corr,
1757                    body,
1758                )
1759                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1760                frames
1761                    .try_send(frame)
1762                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1763            }
1764            state.subscribers.insert(
1765                (connection_id, corr),
1766                SpawnSubscriber {
1767                    version,
1768                    frames,
1769                    lagged: Some(lagged),
1770                },
1771            );
1772        }
1773        // The lagged terminal is sent here, by the forwarder, rather than by
1774        // the emitter: at the moment of the drop the subscriber's own channel
1775        // is full, and writing to the connection sink directly from the emitter
1776        // would put the Error AHEAD of the events still queued in that channel
1777        // (and the emitter holds the feed lock, so it cannot await the sink).
1778        // Dropping the subscriber drops the only sender, so `recv` drains every
1779        // queued event and then returns `None`; only then is the Error sent, so
1780        // the client sees each event it can keep, then the reason it was cut.
1781        // Cancel and connection removal drop the oneshot unsent, so they end
1782        // the stream with no Error.
1783        tokio::spawn(async move {
1784            while let Some(frame) = receiver.recv().await {
1785                if sink.send(frame).await.is_err() {
1786                    return;
1787                }
1788            }
1789            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1790                return;
1791            };
1792            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1793                Ok(frame) => {
1794                    let _ = sink.send(frame).await;
1795                }
1796                Err(error) => {
1797                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1798                }
1799            }
1800        });
1801        Ok(())
1802    }
1803
1804    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1805        let Some(subscriber) = self
1806            .0
1807            .lock()
1808            .unwrap_or_else(|p| p.into_inner())
1809            .subscribers
1810            .remove(&(connection_id, corr))
1811        else {
1812            return false;
1813        };
1814        if let Ok(frame) = Frame::build_with_version(
1815            subscriber.version,
1816            FrameType::StreamEnd,
1817            control_flags(),
1818            0,
1819            0,
1820            corr,
1821            Vec::new(),
1822        ) {
1823            tokio::spawn(async move {
1824                let _ = subscriber.frames.send(frame).await;
1825            });
1826        }
1827        true
1828    }
1829
1830    fn remove_connection(&self, connection_id: ConnectionId) {
1831        self.0
1832            .lock()
1833            .unwrap_or_else(|p| p.into_inner())
1834            .subscribers
1835            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1836    }
1837
1838    #[cfg(any(test, feature = "test-support"))]
1839    fn set_capacity(&self, capacity: usize) {
1840        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1841    }
1842
1843    #[cfg(any(test, feature = "test-support"))]
1844    fn subscriber_count(&self) -> usize {
1845        self.0
1846            .lock()
1847            .unwrap_or_else(|p| p.into_inner())
1848            .subscribers
1849            .len()
1850    }
1851}
1852
1853/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1854/// The terminal Error a lagged spawn subscriber receives after its queued events.
1855fn spawn_subscriber_lagged_frame(
1856    version: u8,
1857    corr: u64,
1858    first_undelivered: SpawnCursor,
1859) -> Result<Frame, String> {
1860    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1861    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1862        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1863        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1864            .to_string(),
1865        detail: Some(serde_json::json!({
1866            "first_undelivered_cursor": first_undelivered
1867        })),
1868    })
1869    .map_err(|error| error.to_string())?;
1870    Frame::build_with_version(
1871        version,
1872        FrameType::Error,
1873        crate::router::daemon_error_flags(),
1874        0,
1875        0,
1876        corr,
1877        body,
1878    )
1879    .map_err(|error| error.to_string())
1880}
1881
1882pub trait ModuleProcessLiveness: Send + Sync {
1883    fn process_live(&self, module_id: &str) -> Option<bool>;
1884
1885    /// Whether the supervisor is replacing this module's process right now: an
1886    /// operator restart or reload, a health restart, or a crash respawn whose
1887    /// backoff is running. A module in that state is not live, but a consumer
1888    /// refused now should retry shortly rather than treat the target as gone.
1889    /// Stopped, failed, and disabled modules are not replacing.
1890    fn process_replacing(&self, _module_id: &str) -> bool {
1891        false
1892    }
1893}
1894
1895/// Shared process-liveness registry keyed by supervised `module_id`.
1896#[derive(Debug, Clone, Default)]
1897pub struct SupervisorProcessLiveness {
1898    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1899}
1900
1901impl SupervisorProcessLiveness {
1902    pub fn new() -> Self {
1903        Self::default()
1904    }
1905
1906    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1907        let mut snapshots = self
1908            .snapshots
1909            .lock()
1910            .unwrap_or_else(|poisoned| poisoned.into_inner());
1911        snapshots.insert(module_id, snapshot);
1912    }
1913
1914    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1915        let mut snapshots = self
1916            .snapshots
1917            .lock()
1918            .unwrap_or_else(|poisoned| poisoned.into_inner());
1919        let is_current = snapshots
1920            .get(module_id)
1921            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1922            .unwrap_or(false);
1923        if is_current {
1924            snapshots.remove(module_id);
1925        }
1926    }
1927}
1928
1929impl ModuleProcessLiveness for SupervisorProcessLiveness {
1930    fn process_live(&self, module_id: &str) -> Option<bool> {
1931        let snapshot = {
1932            let snapshots = self
1933                .snapshots
1934                .lock()
1935                .unwrap_or_else(|poisoned| poisoned.into_inner());
1936            snapshots.get(module_id).cloned()
1937        }?;
1938        let snapshot = snapshot
1939            .lock()
1940            .unwrap_or_else(|poisoned| poisoned.into_inner());
1941        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1942    }
1943
1944    fn process_replacing(&self, module_id: &str) -> bool {
1945        let Some(snapshot) = self
1946            .snapshots
1947            .lock()
1948            .unwrap_or_else(|poisoned| poisoned.into_inner())
1949            .get(module_id)
1950            .cloned()
1951        else {
1952            return false;
1953        };
1954        let snapshot = snapshot
1955            .lock()
1956            .unwrap_or_else(|poisoned| poisoned.into_inner());
1957        snapshot.enabled
1958            && match snapshot.state {
1959                ModuleState::Restarting => true,
1960                ModuleState::Draining => snapshot.draining_to_replace,
1961                ModuleState::Starting
1962                | ModuleState::Running
1963                | ModuleState::Unresponsive
1964                | ModuleState::Stopped
1965                | ModuleState::Failed
1966                | ModuleState::Disabled => false,
1967            }
1968    }
1969}
1970
1971#[cfg(test)]
1972#[derive(Debug, Default)]
1973struct ReloadExitRecordGate {
1974    reached: tokio::sync::Notify,
1975    resume: tokio::sync::Notify,
1976}
1977
1978#[derive(Debug, Clone, Copy)]
1979enum RespawnKind {
1980    Spawn,
1981    Reload,
1982}
1983
1984#[derive(Debug, Clone, Copy)]
1985struct PendingRespawn {
1986    deadline: Instant,
1987    kind: RespawnKind,
1988    daemon_initiated: bool,
1989}
1990
1991type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1992
1993#[derive(Debug, Clone)]
1994struct SupervisorRuntimeConfig {
1995    registry: Arc<Registry>,
1996    /// Retained across an operation's deferred backoff, until its child starts.
1997    recovery_is_daemon: bool,
1998    start_requester: Option<Principal>,
1999    pending_activity_restart: bool,
2000    provenance_probe: ExecutableIdentityProbe,
2001    /// Restart operations hand their backoff to the loop, which keeps serving commands.
2002    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2003    /// A reload acknowledges completion only after its replacement registers.
2004    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2005    restart_policy: RestartPolicy,
2006    register_timeout: Duration,
2007    /// This module's RESOLVED drain budget: per-module config when present,
2008    /// else `default_drain_timeout`.
2009    drain_timeout: Duration,
2010    /// Shared with the status handle so the attested value changes atomically
2011    /// when a rescan updates the running drain policy.
2012    effective_drain_timeout: Arc<Mutex<Duration>>,
2013    /// The supervisor-wide fallback, kept so a configuration update that
2014    /// REMOVES the per-module override can re-resolve to it.
2015    default_drain_timeout: Duration,
2016    health: HealthConfig,
2017    connection_file_path: Option<PathBuf>,
2018    capture_logs_dir: Option<PathBuf>,
2019    forwarding: Option<Arc<ForwardingTable>>,
2020    /// The shared handle, so every spawn path (initial, restart, reload) records the
2021    /// reserved-module launch nonce the HELLO verifier checks against.
2022    supervisor_handle: Option<SupervisorHandle>,
2023    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
2024    /// status queries.
2025    ///
2026    /// One ring per module, held across every respawn. The lines explaining an exit
2027    /// are written BEFORE that exit, so a ring recreated per process would be empty
2028    /// exactly when it is asked for.
2029    stderr_ring: Arc<Mutex<StderrRing>>,
2030    terminal_ring: Arc<Mutex<TerminalRing>>,
2031    spawn_events: SpawnEventFeed,
2032    child_roster: ChildRoster,
2033    #[cfg(target_os = "linux")]
2034    cgroup_placement: Option<subc_cgroup::Placement>,
2035    #[cfg(test)]
2036    test_seed_stale_facts_before_enable_spawn: bool,
2037    #[cfg(test)]
2038    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2039}
2040
2041impl SupervisorRuntimeConfig {
2042    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2043        FailureContext {
2044            daemon_initiated,
2045            shutdown: self.child_roster.shutdown_flag(),
2046        }
2047    }
2048}
2049
2050#[derive(Debug, Clone, PartialEq, Eq)]
2051struct SupervisedConfiguration {
2052    spec: ModuleSpec,
2053    health: HealthConfig,
2054}
2055
2056/// Shared daemon lookup table for supervised module handles.
2057///
2058/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2059/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2060/// launch nonces recorded at spawn are checked by the same daemon instance.
2061#[derive(Debug, Clone, Default)]
2062pub struct SupervisorHandle {
2063    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2064    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2065    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2066    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2067    /// Module ids the supervisor has taken on. An id is added BEFORE the
2068    /// module's first process is spawned and removed only when the module
2069    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2070    /// the keys of `modules`.
2071    ///
2072    /// `modules` cannot answer "is this module configured?" on its own: a
2073    /// [`SupervisedModule`] only exists once its process has been spawned, and
2074    /// a fast child can connect, register, sync its scopes and ask about them
2075    /// before the supervisor has inserted it. Answering "not configured" in that
2076    /// gap makes scope admission refuse with the terminal "will never sync"
2077    /// instead of the retryable "has not synced yet".
2078    configured_ids: Arc<Mutex<HashSet<String>>>,
2079    spawn_events: SpawnEventFeed,
2080    /// The current expected launch nonce for each reserved module_id. Set when the
2081    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2082    /// non-reserved module never has an entry here and is never nonce-checked.
2083    /// Reserved module ids and the nonce that authorizes their next HELLO.
2084    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2085    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2086    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2087    /// had NO entry and admitted anyone: the reservation protected the nonce
2088    /// holder, not the NAME (found live by CKCRED's canary probe registering
2089    /// against a reserved scratch id).
2090    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2091    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2092    ///
2093    /// This is deliberately in-memory only: subc is state-free across daemon
2094    /// restarts, and the tombstone only explains the hours-after-removal window
2095    /// while this executing daemon is still alive. Do not persist it in a store.
2096    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2097    /// The current launch nonce for every supervised spawn. This is separate from
2098    /// reserved_nonces because consumer route.open attestation applies to all spawned
2099    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2100    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2101    /// Reserved namespace prefixes mapped to the supervised owner module whose
2102    /// current spawn nonce authorizes HELLO claims below the prefix.
2103    ///
2104    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2105    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2106    /// accidental collisions and lower-trust processes from squatting protected
2107    /// namespaces.
2108    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2109    /// Blue/green swaps in progress, by module id. An entry exists from just
2110    /// before the candidate process is spawned until the swap has failed, or
2111    /// has cut over and the old process is gone. While it exists, HELLO for the
2112    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2113    /// consumer attestation accepts both processes' nonces.
2114    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2115    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2116    promotion_observer: PromotionObserverSlot,
2117    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2118    /// this daemon-wide ordering, a rescan could retire or update a module while a
2119    /// concurrent reload still held its old handle and launch specification.
2120    operation_lock: Arc<AsyncMutex<()>>,
2121}
2122
2123/// Told when a swap has promoted its candidate to be the module's active
2124/// registration.
2125///
2126/// An ordinary HELLO runs the control plane's registration side effects (the
2127/// capability cache, the deny census, the requirement recompute) as it
2128/// registers. A swap candidate's HELLO does not, because it is not routable;
2129/// promotion is when those must run instead, and promotion happens in the
2130/// supervisor, which has no other way into the control handler.
2131pub(crate) trait SwapPromotionObserver: Send + Sync {
2132    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2133}
2134
2135/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2136/// control handler) owns this handle, so a strong reference back would be a
2137/// cycle that keeps both alive.
2138#[derive(Clone, Default)]
2139struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2140
2141impl fmt::Debug for PromotionObserverSlot {
2142    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2143        f.write_str("PromotionObserverSlot")
2144    }
2145}
2146
2147/// The nonces of one open swap.
2148#[derive(Debug, Clone)]
2149struct OpenSwap {
2150    /// The launch nonce minted for the candidate process. It is the swap
2151    /// token: the only thing that admits a HELLO into the candidate slot.
2152    candidate_nonce: String,
2153    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2154    /// here because cutover moves the module's recorded spawn nonce to the
2155    /// candidate while the incumbent is still draining and its consumers are
2156    /// still attesting with this one.
2157    incumbent_nonce: Option<String>,
2158    /// Set once a HELLO has been admitted with the swap token, so the token
2159    /// admits one registration and cannot be replayed after cutover empties
2160    /// the candidate slot.
2161    candidate_admitted: bool,
2162}
2163
2164/// What the swap gate says about a HELLO. See
2165/// [`SupervisorHandle::swap_hello_admission`].
2166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2167pub(crate) enum SwapHelloAdmission {
2168    /// No swap is open for the id (or the HELLO carries the incumbent's own
2169    /// nonce); the ordinary gates decide.
2170    NotSwapping,
2171    /// The HELLO carries the swap token: register it into the candidate slot.
2172    Candidate,
2173    /// A swap is open and the HELLO carries a nonce the supervisor did not
2174    /// mint for this id, no nonce, or a token already used.
2175    Refused,
2176}
2177
2178#[derive(Debug, Clone, PartialEq, Eq)]
2179pub(crate) enum ReservedHelloRejection {
2180    Exact {
2181        module_id: String,
2182    },
2183    Prefix {
2184        prefix: String,
2185        owner_module_id: String,
2186    },
2187}
2188
2189/// Identity facts established by the reserved-module policy for one HELLO.
2190#[derive(Debug, Default)]
2191pub(crate) struct ReservedHelloVerification {
2192    pub(crate) nonce_verified: bool,
2193    pub(crate) prefix_owner: Option<String>,
2194}
2195
2196#[derive(Debug)]
2197struct DaemonRestartIntent {
2198    expires: Option<Duration>,
2199    window: Duration,
2200}
2201
2202impl Default for DaemonRestartIntent {
2203    fn default() -> Self {
2204        Self {
2205            expires: None,
2206            window: Duration::from_secs(120),
2207        }
2208    }
2209}
2210
2211impl SupervisorHandle {
2212    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2213        let mut intent = self
2214            .restart_intent
2215            .lock()
2216            .unwrap_or_else(|p| p.into_inner());
2217        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2218        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2219    }
2220
2221    #[cfg(feature = "test-support")]
2222    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2223        self.restart_intent
2224            .lock()
2225            .unwrap_or_else(|p| p.into_inner())
2226            .window = window;
2227    }
2228
2229    #[cfg(unix)]
2230    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2231        let expiry = self
2232            .restart_intent
2233            .lock()
2234            .unwrap_or_else(|p| p.into_inner())
2235            .expires
2236            .take();
2237        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2238            RouteCloseReason::Restart
2239        } else {
2240            RouteCloseReason::Disable
2241        }
2242    }
2243
2244    pub fn new() -> Self {
2245        Self::default()
2246    }
2247
2248    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2249        self.spawn_events.snapshot()
2250    }
2251
2252    pub(crate) fn subscribe_spawns(
2253        &self,
2254        connection_id: ConnectionId,
2255        corr: u64,
2256        version: u8,
2257        since: Option<SpawnCursor>,
2258        sink: FrameSink,
2259    ) -> Result<(), SpawnSubscribeRefusal> {
2260        self.spawn_events
2261            .subscribe(connection_id, corr, version, since, sink)
2262    }
2263
2264    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2265        self.spawn_events.cancel(connection_id, corr)
2266    }
2267
2268    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2269        self.spawn_events.remove_connection(connection_id);
2270    }
2271
2272    #[cfg(any(test, feature = "test-support"))]
2273    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2274        assert!(capacity > 0, "spawn event capacity must be non-zero");
2275        self.spawn_events.set_capacity(capacity);
2276    }
2277
2278    #[cfg(any(test, feature = "test-support"))]
2279    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2280        self.spawn_events.subscriber_count()
2281    }
2282
2283    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2284    /// a respawn invalidates stale consumer identities.
2285    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2286        self.spawn_nonces
2287            .lock()
2288            .unwrap_or_else(|poisoned| poisoned.into_inner())
2289            .insert(module_id.to_string(), nonce);
2290    }
2291
2292    /// Record the launch nonce expected from the next HELLO for a reserved module,
2293    /// replacing any prior nonce (a respawn invalidates the previous one).
2294    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2295        self.reserved_nonces
2296            .lock()
2297            .unwrap_or_else(|poisoned| poisoned.into_inner())
2298            .insert(module_id.to_string(), Some(nonce));
2299    }
2300
2301    /// Record namespace prefixes owned by a supervised module.
2302    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2303        let mut owners = self
2304            .reserved_prefix_owners
2305            .lock()
2306            .unwrap_or_else(|poisoned| poisoned.into_inner());
2307        owners.retain(|_, owner| owner != owner_module_id);
2308        for prefix in prefixes {
2309            owners.insert(prefix.clone(), owner_module_id.to_string());
2310        }
2311    }
2312
2313    /// The launch nonce most recently minted for a module's spawn, if any.
2314    #[cfg(test)]
2315    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2316        self.spawn_nonces
2317            .lock()
2318            .unwrap_or_else(|poisoned| poisoned.into_inner())
2319            .get(module_id)
2320            .cloned()
2321    }
2322
2323    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2324        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2325        let spawn_nonce = self
2326            .spawn_nonces
2327            .lock()
2328            .unwrap_or_else(|poisoned| poisoned.into_inner())
2329            .get(&spec.module_id)
2330            .cloned();
2331        let mut reserved_nonces = self
2332            .reserved_nonces
2333            .lock()
2334            .unwrap_or_else(|poisoned| poisoned.into_inner());
2335        if spec.reserved {
2336            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2337            // reserved name whose module has never spawned has no legitimate
2338            // holder, and the entry's absence is what used to leave the name
2339            // open to the first claimant.
2340            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2341        }
2342        drop(reserved_nonces);
2343        // A later unreserved declaration must not silently unreserve an id that
2344        // was retained after its reserved configuration was removed. The explicit
2345        // release ceremony is the only operation that retires that gate.
2346        self.removal_tombstones
2347            .lock()
2348            .unwrap_or_else(|poisoned| poisoned.into_inner())
2349            .remove(&spec.module_id);
2350    }
2351
2352    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2353    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2354    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2355    /// with no matching prefix are always authorized.
2356    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2357        self.reserved_hello_verification(module_id, presented)
2358            .is_ok()
2359    }
2360
2361    /// Whether HELLO passed a nonce check that was actually required by reserved
2362    /// module policy, or the refusal explaining why a required check failed.
2363    pub(crate) fn reserved_hello_verification(
2364        &self,
2365        module_id: &str,
2366        presented: Option<&str>,
2367    ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2368        let nonces = self
2369            .reserved_nonces
2370            .lock()
2371            .unwrap_or_else(|poisoned| poisoned.into_inner());
2372        if let Some(expected) = nonces.get(module_id) {
2373            // `None` = reserved with no legitimate holder: refuse every
2374            // presentation, because no process can hold a nonce that was never
2375            // minted. Only a real minted nonce admits, in constant time.
2376            let authorized = match expected {
2377                Some(expected) => {
2378                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2379                }
2380                None => false,
2381            };
2382            if authorized {
2383                return Ok(ReservedHelloVerification {
2384                    nonce_verified: true,
2385                    prefix_owner: None,
2386                });
2387            }
2388            return Err(ReservedHelloRejection::Exact {
2389                module_id: module_id.to_string(),
2390            });
2391        }
2392        drop(nonces);
2393
2394        let matched_prefix = self
2395            .reserved_prefix_owners
2396            .lock()
2397            .unwrap_or_else(|poisoned| poisoned.into_inner())
2398            .iter()
2399            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2400            .max_by_key(|(prefix, _)| prefix.len())
2401            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2402        let Some((prefix, owner_module_id)) = matched_prefix else {
2403            return Ok(ReservedHelloVerification::default());
2404        };
2405
2406        let authorized = presented.is_some_and(|presented| {
2407            self.spawn_nonces
2408                .lock()
2409                .unwrap_or_else(|poisoned| poisoned.into_inner())
2410                .get(&owner_module_id)
2411                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2412                // While the owner is being swapped, children started by
2413                // either of its two processes hold that process's nonce.
2414                || self.swap_nonce_matches(&owner_module_id, presented)
2415        });
2416        if authorized {
2417            Ok(ReservedHelloVerification {
2418                nonce_verified: true,
2419                prefix_owner: Some(owner_module_id),
2420            })
2421        } else {
2422            Err(ReservedHelloRejection::Prefix {
2423                prefix,
2424                owner_module_id,
2425            })
2426        }
2427    }
2428
2429    /// Whether a consumer connection proved it came from a daemon-spawned module.
2430    ///
2431    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2432    /// accepted only for module ids the supervisor has spawned.
2433    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2434        if presented.is_empty() {
2435            return false;
2436        }
2437        let nonces = self
2438            .spawn_nonces
2439            .lock()
2440            .unwrap_or_else(|poisoned| poisoned.into_inner());
2441        let current = nonces
2442            .get(module_id)
2443            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2444        drop(nonces);
2445        // During a swap two processes of the module are alive, and a consumer
2446        // started by either one presents that process's nonce. Accepting only
2447        // the recorded one would fail the incumbent's consumers for the whole
2448        // overlap once cutover moves the record to the candidate.
2449        current || self.swap_nonce_matches(module_id, presented)
2450    }
2451
2452    /// Whether `presented` is either nonce of an open swap for `module_id`.
2453    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2454        let swaps = self
2455            .swaps
2456            .lock()
2457            .unwrap_or_else(|poisoned| poisoned.into_inner());
2458        swaps.get(module_id).is_some_and(|swap| {
2459            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2460                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2461                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2462                })
2463        })
2464    }
2465
2466    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2467    /// Called before the candidate process exists.
2468    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2469        let incumbent_nonce = self
2470            .spawn_nonces
2471            .lock()
2472            .unwrap_or_else(|poisoned| poisoned.into_inner())
2473            .get(module_id)
2474            .cloned();
2475        self.swaps
2476            .lock()
2477            .unwrap_or_else(|poisoned| poisoned.into_inner())
2478            .insert(
2479                module_id.to_string(),
2480                OpenSwap {
2481                    candidate_nonce,
2482                    incumbent_nonce,
2483                    candidate_admitted: false,
2484                },
2485            );
2486    }
2487
2488    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2489    /// the module's recorded one.
2490    pub(crate) fn close_swap(&self, module_id: &str) {
2491        self.swaps
2492            .lock()
2493            .unwrap_or_else(|poisoned| poisoned.into_inner())
2494            .remove(module_id);
2495    }
2496
2497    /// Install the observer told about swap promotions, replacing any earlier
2498    /// one.
2499    pub(crate) fn set_swap_promotion_observer(
2500        &self,
2501        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2502    ) {
2503        *self
2504            .promotion_observer
2505            .0
2506            .lock()
2507            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2508    }
2509
2510    /// Tell the installed observer, if it is still alive, that a swap promoted
2511    /// `registration`.
2512    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2513        let observer = self
2514            .promotion_observer
2515            .0
2516            .lock()
2517            .unwrap_or_else(|poisoned| poisoned.into_inner())
2518            .as_ref()
2519            .and_then(std::sync::Weak::upgrade);
2520        if let Some(observer) = observer {
2521            observer.swap_promoted(registration);
2522        }
2523    }
2524
2525    /// Whether a swap is open for `module_id`.
2526    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2527        self.swaps
2528            .lock()
2529            .unwrap_or_else(|poisoned| poisoned.into_inner())
2530            .contains_key(module_id)
2531    }
2532
2533    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2534    /// respawn would, once cutover has made the candidate the module's process.
2535    /// The swap stays open so the incumbent's nonce keeps attesting until the
2536    /// incumbent has drained and exited.
2537    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2538        let candidate_nonce = self
2539            .swaps
2540            .lock()
2541            .unwrap_or_else(|poisoned| poisoned.into_inner())
2542            .get(module_id)
2543            .map(|swap| swap.candidate_nonce.clone());
2544        let Some(nonce) = candidate_nonce else {
2545            return;
2546        };
2547        self.set_spawn_nonce(module_id, nonce.clone());
2548        if reserved {
2549            self.set_reserved_nonce(module_id, nonce);
2550        }
2551    }
2552
2553    /// The swap gate for a HELLO claiming `module_id`.
2554    ///
2555    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2556    /// presents the candidate nonce, which the reserved gate (holding the
2557    /// incumbent's nonce) would refuse as `reserved_module` before swap
2558    /// admission was ever reached. And it applies to unreserved ids too: for an
2559    /// unreserved id the only thing that ever stopped a second process claiming
2560    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2561    /// refusal a swap lifts for its candidate.
2562    ///
2563    /// The incumbent's own nonce falls through to the ordinary gates, which
2564    /// treat it as they always have (a live incumbent is refused as a
2565    /// duplicate). Anything else while a swap is open is refused, including an
2566    /// absent nonce.
2567    pub(crate) fn swap_hello_admission(
2568        &self,
2569        module_id: &str,
2570        presented: Option<&str>,
2571    ) -> SwapHelloAdmission {
2572        let swaps = self
2573            .swaps
2574            .lock()
2575            .unwrap_or_else(|poisoned| poisoned.into_inner());
2576        let Some(swap) = swaps.get(module_id) else {
2577            return SwapHelloAdmission::NotSwapping;
2578        };
2579        let Some(presented) = presented else {
2580            return SwapHelloAdmission::Refused;
2581        };
2582        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2583            return if swap.candidate_admitted {
2584                SwapHelloAdmission::Refused
2585            } else {
2586                SwapHelloAdmission::Candidate
2587            };
2588        }
2589        if swap
2590            .incumbent_nonce
2591            .as_deref()
2592            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2593        {
2594            return SwapHelloAdmission::NotSwapping;
2595        }
2596        SwapHelloAdmission::Refused
2597    }
2598
2599    /// Record that the swap token has registered a candidate, so it admits no
2600    /// second HELLO.
2601    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2602        if let Some(swap) = self
2603            .swaps
2604            .lock()
2605            .unwrap_or_else(|poisoned| poisoned.into_inner())
2606            .get_mut(module_id)
2607        {
2608            swap.candidate_admitted = true;
2609        }
2610    }
2611
2612    /// Test/support lookup for the current launch nonce of a supervised spawn.
2613    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2614        self.spawn_nonces
2615            .lock()
2616            .unwrap_or_else(|poisoned| poisoned.into_inner())
2617            .get(module_id)
2618            .cloned()
2619    }
2620
2621    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2622    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2623        self.reserved_nonces
2624            .lock()
2625            .unwrap_or_else(|poisoned| poisoned.into_inner())
2626            .get(module_id)
2627            .cloned()
2628            .flatten()
2629    }
2630
2631    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2632        // Normally already marked before the process was spawned; marking here
2633        // too keeps `configured_ids` a superset of the roster for any caller
2634        // that inserts a module directly.
2635        self.mark_configured(module.module_id());
2636        let mut modules = self
2637            .modules
2638            .lock()
2639            .unwrap_or_else(|poisoned| poisoned.into_inner());
2640        modules.insert(module.module_id().to_string(), module)
2641    }
2642
2643    /// Record that the supervisor has taken on `module_id`. Called before the
2644    /// module's first process is spawned, so that by the time that process can
2645    /// register, [`Self::is_configured`] already answers true.
2646    fn mark_configured(&self, module_id: &str) {
2647        self.configured_ids
2648            .lock()
2649            .unwrap_or_else(|poisoned| poisoned.into_inner())
2650            .insert(module_id.to_string());
2651    }
2652
2653    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2654    /// before it was ever put on the roster. A module already on the roster
2655    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2656    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2657        let modules = self
2658            .modules
2659            .lock()
2660            .unwrap_or_else(|poisoned| poisoned.into_inner());
2661        if !modules.contains_key(module_id) {
2662            self.configured_ids
2663                .lock()
2664                .unwrap_or_else(|poisoned| poisoned.into_inner())
2665                .remove(module_id);
2666        }
2667    }
2668
2669    /// Whether `module_id` is a module this daemon supervises: on the roster,
2670    /// or about to be (its process is being spawned right now).
2671    ///
2672    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2673    /// for scopes: a supervised module's process can register and sync before
2674    /// [`Self::get`] can return it, and in that window it is still a module
2675    /// that will sync, not one that never will.
2676    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2677        self.configured_ids
2678            .lock()
2679            .unwrap_or_else(|poisoned| poisoned.into_inner())
2680            .contains(module_id)
2681    }
2682
2683    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2684        let modules = self
2685            .modules
2686            .lock()
2687            .unwrap_or_else(|poisoned| poisoned.into_inner());
2688        modules.get(module_id).cloned()
2689    }
2690
2691    pub(crate) fn record_late_health_answer(
2692        &self,
2693        module_id: &str,
2694        latency_ms: u64,
2695    ) -> Result<bool, SuperviseError> {
2696        let Some(module) = self.get(module_id) else {
2697            return Ok(false);
2698        };
2699        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2700            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2701            state.health.last_late_answer_latency_ms = Some(latency_ms);
2702            // A late answer is an answer: the module served the probe, just past
2703            // the deadline. Leaving the miss streak in place while logging
2704            // "proves the module is alive" is how a CPU-starved module that
2705            // answers every probe a few seconds late still marches to the
2706            // threshold and gets killed — the exact kill class `NoAnswer` is
2707            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2708            // is degradation, and degradation reports; it does not restart.
2709            state.health.consecutive_failures = 0;
2710        })?;
2711        Ok(true)
2712    }
2713
2714    /// Arm the one-shot marker for the module process that this caller
2715    /// deliberately initiated severance against. Generic connection teardown
2716    /// must not call this:
2717    /// a surviving process would otherwise retain an exemption for a later
2718    /// genuine crash.
2719    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2720        let Some(module) = self.get(module_id) else {
2721            return Ok(false);
2722        };
2723        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2724        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2725            return Ok(false);
2726        };
2727        drop(snapshot);
2728        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2729    }
2730
2731    pub fn list(&self) -> Vec<SupervisedModule> {
2732        let modules = self
2733            .modules
2734            .lock()
2735            .unwrap_or_else(|poisoned| poisoned.into_inner());
2736        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2737        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2738        modules
2739    }
2740
2741    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2742        self.activity_images
2743            .lock()
2744            .unwrap_or_else(|p| p.into_inner())
2745            .remove(module_id);
2746        self.spawn_nonces
2747            .lock()
2748            .unwrap_or_else(|poisoned| poisoned.into_inner())
2749            .remove(module_id);
2750        self.close_swap(module_id);
2751        let mut reserved_nonces = self
2752            .reserved_nonces
2753            .lock()
2754            .unwrap_or_else(|poisoned| poisoned.into_inner());
2755        if reserved_nonces.contains_key(module_id) {
2756            // The old nonce must die with the removed process, but the exact-id
2757            // gate remains until an operator explicitly releases it.
2758            reserved_nonces.insert(module_id.to_string(), None);
2759        }
2760        drop(reserved_nonces);
2761        self.reserved_prefix_owners
2762            .lock()
2763            .unwrap_or_else(|poisoned| poisoned.into_inner())
2764            .retain(|_, owner| owner != module_id);
2765        let removed = self
2766            .modules
2767            .lock()
2768            .unwrap_or_else(|poisoned| poisoned.into_inner())
2769            .remove(module_id);
2770        self.configured_ids
2771            .lock()
2772            .unwrap_or_else(|poisoned| poisoned.into_inner())
2773            .remove(module_id);
2774        removed
2775    }
2776
2777    /// Remember a module removed by a non-preview rescan so route.open can
2778    /// distinguish that intentional removal from an unknown id.
2779    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2780        self.removal_tombstones
2781            .lock()
2782            .unwrap_or_else(|poisoned| poisoned.into_inner())
2783            .insert(module_id.to_string(), unix_ms_now());
2784    }
2785
2786    /// Return how long ago a rescan removed this module in milliseconds.
2787    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2788        self.removal_tombstones
2789            .lock()
2790            .unwrap_or_else(|poisoned| poisoned.into_inner())
2791            .get(module_id)
2792            .copied()
2793            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2794    }
2795
2796    /// Retire a reserved-id gate only after its module has left supervision.
2797    ///
2798    /// A retained gate has no live nonce (`None`), so releasing any other entry
2799    /// would weaken a currently configured or otherwise active reservation.
2800    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2801        if self.get(module_id).is_some() {
2802            return false;
2803        }
2804        let mut reserved_nonces = self
2805            .reserved_nonces
2806            .lock()
2807            .unwrap_or_else(|poisoned| poisoned.into_inner());
2808        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2809            return false;
2810        }
2811        reserved_nonces.remove(module_id);
2812        true
2813    }
2814
2815    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2816        Arc::clone(&self.operation_lock)
2817    }
2818}
2819
2820/// Process supervisor for subc-owned singleton modules.
2821#[derive(Debug, Clone)]
2822pub struct Supervisor {
2823    registry: Arc<Registry>,
2824    restart_policy: RestartPolicy,
2825    drain_timeout: Duration,
2826    connection_file_path: Option<PathBuf>,
2827    capture_logs_dir: Option<PathBuf>,
2828    forwarding: Option<Arc<ForwardingTable>>,
2829    process_liveness: Arc<SupervisorProcessLiveness>,
2830    supervisor_handle: Option<SupervisorHandle>,
2831    health: HealthConfig,
2832    daemon_start_clock: crate::clock::StartClock,
2833    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2834    spawn_events: SpawnEventFeed,
2835    provenance_probe: ExecutableIdentityProbe,
2836    /// Every process spawned through this supervisor (and its clones) and not
2837    /// yet reaped, so daemon shutdown can end them.
2838    child_roster: ChildRoster,
2839    #[cfg(target_os = "linux")]
2840    cgroup_placement: Option<subc_cgroup::Placement>,
2841    #[cfg(test)]
2842    test_after_first_spawn: AfterFirstSpawnHook,
2843}
2844
2845/// A roster entry whose monitor and first process have not started yet.
2846/// Keeping the command receiver here lets discovery serve status immediately
2847/// while start/stop requests wait until bootstrap has published the connection file.
2848pub(crate) struct RegisteredConfiguredModule {
2849    spec: ModuleSpec,
2850    runtime: SupervisorRuntimeConfig,
2851    module: SupervisedModule,
2852    commands: mpsc::Receiver<SupervisorCommand>,
2853}
2854
2855/// Test-only hook run on the path that takes on a new module, right after its
2856/// first `spawn_child` returns (the process exists and could already be
2857/// registering) and before that process is handed to the module's supervise
2858/// loop and put on the roster. Lets a test observe what a fast child would see
2859/// in that window without racing a real one.
2860#[cfg(test)]
2861#[derive(Clone, Default)]
2862struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2863
2864#[cfg(test)]
2865type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2866
2867#[cfg(test)]
2868impl fmt::Debug for AfterFirstSpawnHook {
2869    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2870        f.write_str("AfterFirstSpawnHook")
2871    }
2872}
2873
2874#[cfg(test)]
2875impl AfterFirstSpawnHook {
2876    fn run(&self, module_id: &str) {
2877        if let Some(hook) = &self.0 {
2878            hook(module_id);
2879        }
2880    }
2881}
2882
2883impl Supervisor {
2884    #[cfg(test)]
2885    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2886        let supervisor = Self::new(registry, policy);
2887        #[cfg(target_os = "macos")]
2888        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2889        supervisor
2890    }
2891    /// Verify the trampoline once when configured. Missing private OS support
2892    /// refuses every macOS launch by name but does not stop the daemon's control
2893    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2894    /// entry point; the library must not exec an arbitrary hosting program.
2895    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2896        let path = path.into();
2897        #[cfg(target_os = "macos")]
2898        {
2899            let result = probe_privacy_trampoline(&path).map(|()| path);
2900            if let Err(cause) = &result {
2901                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2902            }
2903            self.child_roster.set_privacy_trampoline(result);
2904        }
2905        #[cfg(not(target_os = "macos"))]
2906        let _ = path;
2907        self
2908    }
2909    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2910    /// capability-requirement log to read.
2911    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2912        self.child_roster.shutdown_flag()
2913    }
2914
2915    /// The first step of an announced daemon shutdown, before the notice and
2916    /// before any connection is closed.
2917    ///
2918    /// Sets the daemon-shutdown flag first: from here on no module is
2919    /// respawned (crash restart, operator restart, or swap), and every child
2920    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2921    /// the module exits on the EOF this shutdown gives it or is signalled by a
2922    /// service manager that kills the whole cgroup. Then writes the journal's
2923    /// shutdown marker, which records the instant and closes this daemon
2924    /// incarnation's stretch of the journal.
2925    #[cfg(unix)]
2926    pub(crate) fn begin_daemon_shutdown(&self) {
2927        self.child_roster.close();
2928        if let Some(journal) = &self.terminal_journal {
2929            journal.stamp_shutdown();
2930        }
2931    }
2932
2933    /// Announce a cut while established connections can still carry replies.
2934    /// These budgets bound the attempt to deliver notices and the wait for
2935    /// routes to drain; neither guarantees delivery or child completion, and
2936    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2937    #[cfg(unix)]
2938    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2939        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2940        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2941        let Some(forwarding) = &self.forwarding else {
2942            return Ok(());
2943        };
2944        let notice_started = Instant::now();
2945        let modules = forwarding
2946            .begin_daemon_drain()
2947            .map_err(SuperviseError::Forwarding)?;
2948        let deadline_ms =
2949            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2950        let shutdown_reason = self
2951            .supervisor_handle
2952            .as_ref()
2953            .map(SupervisorHandle::take_daemon_shutdown_reason)
2954            .unwrap_or(RouteCloseReason::Disable);
2955        let mut modules_told = 0;
2956        let mut notices = tokio::task::JoinSet::new();
2957        let mut drains = Vec::new();
2958        for connection in forwarding
2959            .module_connections()
2960            .map_err(SuperviseError::Forwarding)?
2961        {
2962            let target = ModuleDrainTarget {
2963                endpoint: connection.endpoint,
2964                sink: connection.sink,
2965                negotiated_ver: connection.negotiated_ver,
2966                abandoned_bindings: Vec::new(),
2967                excluded_subscriptions: 0,
2968            };
2969            notices.spawn(async move {
2970                (
2971                    true,
2972                    send_module_draining(
2973                        &connection.module_id,
2974                        shutdown_reason,
2975                        deadline_ms,
2976                        &target,
2977                        true,
2978                    )
2979                    .await,
2980                )
2981            });
2982        }
2983        for (module_id, endpoint) in modules {
2984            let Some(target) = forwarding
2985                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2986                .map_err(SuperviseError::Forwarding)?
2987            else {
2988                continue;
2989            };
2990            let routes = forwarding
2991                .endpoint_routes(target.endpoint)
2992                .map_err(SuperviseError::Forwarding)?;
2993            // Only an announced restart promises that modules are coming back.
2994            // Client route notices retain their existing reconnect semantics.
2995            let mut recipients = Vec::new();
2996            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2997            for route in routes {
2998                let client = route.goodbye_target;
2999                if let Some((_, channels)) = clients
3000                    .iter_mut()
3001                    .find(|(existing, _)| existing.connection_id == client.connection_id)
3002                {
3003                    channels.push(client.channel);
3004                } else {
3005                    let channel = client.channel;
3006                    clients.push((client, vec![channel]));
3007                }
3008            }
3009            for (client, mut channels) in clients {
3010                channels.sort_unstable();
3011                channels.dedup();
3012                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3013                    module_id: module_id.clone(),
3014                    channels,
3015                    reason: RouteCloseReason::Restart,
3016                })
3017                .expect("route closing serializes");
3018                recipients.push((client.sink, client.negotiated_ver, closing));
3019            }
3020            for (sink, version, body) in recipients {
3021                notices.spawn(async move {
3022                    let frame = Frame::build_with_version(
3023                        version,
3024                        FrameType::Push,
3025                        control_flags(),
3026                        0,
3027                        0,
3028                        0,
3029                        body,
3030                    )
3031                    .expect("bounded lifecycle notice frame builds");
3032                    (false, sink.send_flushed(frame).await.is_ok())
3033                });
3034            }
3035            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3036            drains.push((module_id, target.endpoint, gauges));
3037        }
3038        // A quiet forwarding table is not proof that queued notices reached the
3039        // socket. Wait for writer flush acknowledgements before testing quiescence.
3040        let notice_deadline = Instant::now() + NOTICE_BUDGET;
3041        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3042            match result {
3043                Ok((true, true)) => modules_told += 1,
3044                Ok((false, true)) => {}
3045                result => warn!(?result, "daemon shutdown notice delivery failed"),
3046            }
3047        }
3048        notices.abort_all();
3049        info!(
3050            ?shutdown_reason,
3051            modules_told, "daemon shutdown module notices delivered"
3052        );
3053        let notice_elapsed = notice_started.elapsed();
3054        let drain_started = Instant::now();
3055        let deadline = drain_started + DRAIN_BUDGET;
3056        let mut waits = tokio::task::JoinSet::new();
3057        for (module_id, endpoint, gauges) in drains {
3058            let forwarding = Arc::clone(forwarding);
3059            let mut runtime = self.runtime_config();
3060            runtime.health.cadence = Duration::from_millis(100);
3061            waits.spawn(async move {
3062                let wait_result = wait_for_forwarding_quiescence(
3063                    &forwarding,
3064                    &module_id,
3065                    &runtime,
3066                    endpoint,
3067                    deadline,
3068                    &gauges,
3069                    DrainScope::Endpoint(endpoint),
3070                    &mut DrainGaugeTrace::new(&gauges),
3071                )
3072                .await;
3073                // Store the measurement before the task returns: quiescence
3074                // may be reached just as the deadline stops the join loop from
3075                // collecting results. Unfinished or failed waits report false.
3076                forwarding
3077                    .record_daemon_drain_result(
3078                        endpoint,
3079                        drained_after_quiescence_wait(&wait_result),
3080                    )
3081                    .map_err(SuperviseError::Forwarding)?;
3082                wait_result
3083            });
3084        }
3085        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3086            if !matches!(result, Ok(Ok(true))) {
3087                warn!(?result, "daemon shutdown drain did not reach quiescence");
3088            }
3089        }
3090        // Measure only the forwarding drain, not child teardown or the caller's
3091        // signal delivery and process-reaping delays.
3092        info!(
3093            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3094            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3095            "daemon shutdown forwarding drain complete"
3096        );
3097        Ok(())
3098    }
3099
3100    /// The last step of a graceful daemon shutdown, after the notice and the
3101    /// drain: send every registered module a module GOODBYE, the same planned
3102    /// stop signal `ck module stop` gives, then close every connection so each
3103    /// subc module sees EOF and starts its own teardown, then end every
3104    /// supervised child that has not exited
3105    /// by its own deadline (its drain budget, capped). Modules lead their own
3106    /// process groups, so a
3107    /// service manager's group kill no longer reaches them; without this a
3108    /// child that does not stop on EOF (every `protocol: "none"` child, which
3109    /// has no connection) would outlive the daemon. Every wait is bounded (see
3110    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3111    #[cfg(unix)]
3112    pub(crate) async fn end_children_for_daemon_shutdown(
3113        &self,
3114        already_escalated: bool,
3115        escalate: impl std::future::Future<Output = ()>,
3116    ) {
3117        tokio::pin!(escalate);
3118        let mut escalated = already_escalated;
3119        if let Some(forwarding) = &self.forwarding {
3120            let reason = CloseReason::new(
3121                "daemon_shutdown",
3122                "the daemon is exiting after its shutdown notice and drain",
3123            );
3124            if escalated {
3125                // The operator asked to stop waiting: queue the GOODBYEs but
3126                // do not wait for them to be written.
3127                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3128            } else {
3129                tokio::select! {
3130                    biased;
3131                    _ = escalate.as_mut() => {
3132                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3133                        escalated = true;
3134                    }
3135                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3136                }
3137            }
3138            let closed = forwarding.close_all_connections(&reason);
3139            debug!(closed, "closed established connections for daemon shutdown");
3140        }
3141        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3142        // already completed and must not be polled again; the child shutdown
3143        // wait is told it is escalated and gets a future that never fires.
3144        let escalated_here = escalated && !already_escalated;
3145        let remaining_escalate = async move {
3146            if escalated_here {
3147                std::future::pending::<()>().await;
3148            } else {
3149                escalate.await;
3150            }
3151        };
3152        crate::child_roster::end_children_for_daemon_shutdown(
3153            &self.child_roster,
3154            escalated,
3155            remaining_escalate,
3156        )
3157        .await;
3158    }
3159
3160    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3161        Self {
3162            registry,
3163            restart_policy,
3164            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3165            connection_file_path: None,
3166            capture_logs_dir: None,
3167            forwarding: None,
3168            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3169            supervisor_handle: None,
3170            health: HealthConfig::default(),
3171            daemon_start_clock: crate::clock::StartClock::capture(),
3172            terminal_journal: None,
3173            spawn_events: SpawnEventFeed::default(),
3174            provenance_probe: ExecutableIdentityProbe::default(),
3175            child_roster: ChildRoster::default(),
3176            #[cfg(target_os = "linux")]
3177            cgroup_placement: None,
3178            #[cfg(test)]
3179            test_after_first_spawn: AfterFirstSpawnHook::default(),
3180        }
3181    }
3182
3183    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3184        self.drain_timeout = drain_timeout;
3185        self
3186    }
3187
3188    /// The drain budget a module gets when its config entry sets none.
3189    pub(crate) fn default_drain_timeout(&self) -> Duration {
3190        self.drain_timeout
3191    }
3192
3193    pub fn with_process_liveness(
3194        mut self,
3195        process_liveness: Arc<SupervisorProcessLiveness>,
3196    ) -> Self {
3197        self.process_liveness = process_liveness;
3198        self
3199    }
3200
3201    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3202        self.connection_file_path = Some(connection_file_path.into());
3203        self
3204    }
3205
3206    /// Enables daemon-owned capture files for supervised stdout and stderr.
3207    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3208        self.capture_logs_dir = Some(logs_dir.into());
3209        self
3210    }
3211
3212    /// Names this daemon lifetime in spawn events, independently of whether a
3213    /// terminal journal is configured.
3214    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3215        // A millisecond start stamp can repeat after clock rollback or a rapid
3216        // restart. Use the connection file's random daemon_id instead: it already
3217        // identifies this daemon lifetime independently of the wall clock.
3218        self.spawn_events.configure_incarnation(daemon_incarnation);
3219        self
3220    }
3221
3222    /// Enables best-effort history shared by every supervised module. Without
3223    /// it, terminal history is kept only in each module's in-memory ring.
3224    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3225        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3226        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3227            path,
3228            daemon_incarnation,
3229        )));
3230        this
3231    }
3232
3233    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3234        self.child_roster.counters = forwarding.counters();
3235        self.forwarding = Some(forwarding);
3236        self
3237    }
3238
3239    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3240        self.spawn_events = supervisor_handle.spawn_events.clone();
3241        self.supervisor_handle = Some(supervisor_handle);
3242        self
3243    }
3244
3245    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3246        self.health = health;
3247        self
3248    }
3249
3250    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3251    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3252    /// record is kept.
3253    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3254        self.child_roster.record_to(path.into());
3255        self
3256    }
3257
3258    #[cfg(target_os = "linux")]
3259    pub fn with_cgroup_placement(
3260        mut self,
3261        cgroup_placement: Option<subc_cgroup::Placement>,
3262    ) -> Self {
3263        self.child_roster
3264            .set_cgroup_placement(cgroup_placement.clone());
3265        self.cgroup_placement = cgroup_placement;
3266        self
3267    }
3268
3269    /// Spawn `spec.program` and start monitoring it.
3270    ///
3271    /// The child is expected to parse `--subc <connection-file-path>`, read the
3272    /// TCP+key connection file, authenticate to the already-running listener, and
3273    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3274    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3275        validate_spec(&spec)?;
3276        self.establish_identity(&spec);
3277
3278        let runtime = self.runtime_config();
3279        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3280        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3281        let registration_watch = watch_registration(&self.registry, &spec);
3282        let spawned = spawn_child(
3283            &spec,
3284            runtime.connection_file_path.as_deref(),
3285            self.supervisor_handle.as_ref(),
3286            &runtime.stderr_ring,
3287            runtime.capture_logs_dir.as_deref(),
3288            &runtime.child_roster,
3289            #[cfg(target_os = "linux")]
3290            runtime.cgroup_placement.as_ref(),
3291        );
3292        #[cfg(test)]
3293        self.test_after_first_spawn.run(&spec.module_id);
3294        let mut child = match spawned {
3295            Ok(child) => child,
3296            Err(err) => {
3297                // Unlike the configured paths, a failed `spawn` leaves nothing
3298                // on the roster, so the module must not stay marked configured.
3299                self.abandon_unrostered(&spec.module_id);
3300                return Err(err);
3301            }
3302        };
3303        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3304        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3305        if let Some(spawn) = activity_spawn {
3306            spawn.complete(&snapshot);
3307        }
3308
3309        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3310    }
3311
3312    /// Make `spec`'s module count as configured, with its identity gates
3313    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3314    /// exists.
3315    ///
3316    /// Every path that takes on a new module calls this before `spawn_child`.
3317    /// The order is the point: the child can connect, register, sync its
3318    /// scopes and ask about them as soon as it is spawned, and the module is
3319    /// only put on the roster after `spawn_child` returns. Were the mark set
3320    /// with the roster entry, a fast child would see its own owner reported
3321    /// as not configured, and a scoped `route.open` in that window would be
3322    /// refused as terminal `scope_not_live` ("will never sync") instead of
3323    /// retryable `scope_not_synced`.
3324    fn establish_identity(&self, spec: &ModuleSpec) {
3325        if let Some(supervisor_handle) = &self.supervisor_handle {
3326            supervisor_handle.apply_identity_configuration(spec);
3327            supervisor_handle.mark_configured(&spec.module_id);
3328        }
3329    }
3330
3331    /// Take back [`Self::establish_identity`]'s configured mark when the
3332    /// module will not be put on the roster after all.
3333    fn abandon_unrostered(&self, module_id: &str) {
3334        if let Some(supervisor_handle) = &self.supervisor_handle {
3335            supervisor_handle.unmark_configured_unless_rostered(module_id);
3336        }
3337    }
3338
3339    /// Record a freshly spawned first process as running. On failure the
3340    /// module never reaches the roster, so its configured mark is taken back.
3341    fn mark_first_process_running(
3342        &self,
3343        spec: &ModuleSpec,
3344        runtime: &SupervisorRuntimeConfig,
3345        snapshot: &SharedSnapshot,
3346        child: &mut SupervisedChild,
3347    ) -> Result<(), SuperviseError> {
3348        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3349            containment::terminate_blocking(&runtime.child_roster.containment);
3350            self.abandon_unrostered(&spec.module_id);
3351            return Err(err);
3352        }
3353        self.process_liveness
3354            .track(spec.module_id.clone(), Arc::clone(snapshot));
3355        Ok(())
3356    }
3357
3358    /// Start supervising a module declared in daemon configuration.
3359    ///
3360    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3361    /// failures in the supervisor handle so operator-facing `supervisor.list`
3362    /// reflects every configured module while daemon startup continues.
3363    pub fn supervise_configured(
3364        &self,
3365        spec: ModuleSpec,
3366        enabled: bool,
3367    ) -> Result<SupervisedModule, SuperviseError> {
3368        self.supervise_configured_requested(
3369            spec,
3370            enabled,
3371            self.health.clone(),
3372            None,
3373            self.restart_policy,
3374            DEFAULT_REGISTER_TIMEOUT,
3375            None,
3376        )
3377    }
3378
3379    /// Supervise a configured module with its own health, drain, and crash
3380    /// budget. The restart policy is per-module because the config file is:
3381    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3382    /// module that is expensive to restart should not be forced onto the same
3383    /// budget as one that is cheap.
3384    pub fn supervise_configured_with_health(
3385        &self,
3386        spec: ModuleSpec,
3387        enabled: bool,
3388        health: HealthConfig,
3389        drain_timeout_ms: Option<u64>,
3390        restart_policy: RestartPolicy,
3391    ) -> Result<SupervisedModule, SuperviseError> {
3392        self.supervise_configured_requested(
3393            spec,
3394            enabled,
3395            health,
3396            drain_timeout_ms,
3397            restart_policy,
3398            DEFAULT_REGISTER_TIMEOUT,
3399            None,
3400        )
3401    }
3402
3403    #[allow(clippy::too_many_arguments)]
3404    pub(crate) fn supervise_configured_requested(
3405        &self,
3406        spec: ModuleSpec,
3407        enabled: bool,
3408        health: HealthConfig,
3409        drain_timeout_ms: Option<u64>,
3410        restart_policy: RestartPolicy,
3411        register_timeout: Duration,
3412        requester: Option<Principal>,
3413    ) -> Result<SupervisedModule, SuperviseError> {
3414        let registered = self.register_configured_with_health(
3415            spec,
3416            enabled,
3417            health,
3418            drain_timeout_ms,
3419            restart_policy,
3420            register_timeout,
3421        )?;
3422        let daemon_initiated = requester.is_none();
3423        self.start_registered_configured(registered, requester, daemon_initiated)
3424    }
3425
3426    /// Record a configured module as starting or disabled, without launching a
3427    /// process or running its monitor. Bootstrap registers the entire roster
3428    /// before publishing discovery, then activates it once children can read
3429    /// the connection file passed in their `--subc` argument.
3430    pub(crate) fn register_configured_with_health(
3431        &self,
3432        spec: ModuleSpec,
3433        enabled: bool,
3434        health: HealthConfig,
3435        drain_timeout_ms: Option<u64>,
3436        restart_policy: RestartPolicy,
3437        register_timeout: Duration,
3438    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3439        validate_spec(&spec)?;
3440        self.establish_identity(&spec);
3441
3442        let mut runtime = self.runtime_config();
3443        runtime.health = health;
3444        runtime.restart_policy = restart_policy;
3445        runtime.register_timeout = register_timeout;
3446        if let Some(ms) = drain_timeout_ms {
3447            runtime.drain_timeout = Duration::from_millis(ms);
3448            *runtime
3449                .effective_drain_timeout
3450                .lock()
3451                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3452        }
3453        let snapshot = if enabled {
3454            SupervisorSnapshot::starting()
3455        } else {
3456            SupervisorSnapshot::disabled()
3457        };
3458        let snapshot = {
3459            let mut snapshot = snapshot;
3460            if enabled {
3461                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3462                {
3463                    snapshot.waiting_for_previous_pid = Some(pid);
3464                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3465                }
3466            }
3467            snapshot
3468        };
3469        let (module, commands) =
3470            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3471        Ok(RegisteredConfiguredModule {
3472            spec,
3473            runtime,
3474            module,
3475            commands,
3476        })
3477    }
3478
3479    /// Activate a previously registered module. Queued control requests are
3480    /// served by the monitor after the initial configured spawn has settled.
3481    pub(crate) fn start_registered_configured(
3482        &self,
3483        registered: RegisteredConfiguredModule,
3484        requester: Option<Principal>,
3485        daemon_initiated: bool,
3486    ) -> Result<SupervisedModule, SuperviseError> {
3487        let RegisteredConfiguredModule {
3488            spec,
3489            mut runtime,
3490            module,
3491            commands,
3492        } = registered;
3493        runtime.start_requester = requester;
3494        runtime.recovery_is_daemon = daemon_initiated;
3495        let snapshot = &module.inner.snapshot;
3496        let should_spawn = {
3497            let state = lock_snapshot(snapshot)?;
3498            state.enabled && state.waiting_for_previous_pid.is_none()
3499        };
3500        let child = if should_spawn {
3501            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3502            let registration_watch = watch_registration(&self.registry, &spec);
3503            let spawned = spawn_child(
3504                &spec,
3505                runtime.connection_file_path.as_deref(),
3506                self.supervisor_handle.as_ref(),
3507                &runtime.stderr_ring,
3508                runtime.capture_logs_dir.as_deref(),
3509                &runtime.child_roster,
3510                #[cfg(target_os = "linux")]
3511                runtime.cgroup_placement.as_ref(),
3512            );
3513            #[cfg(test)]
3514            self.test_after_first_spawn.run(&spec.module_id);
3515            match spawned {
3516                Ok(mut child) => {
3517                    arm_registration_deadline(
3518                        &mut child,
3519                        runtime.register_timeout,
3520                        registration_watch,
3521                    );
3522                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3523                    if let Some(spawn) = activity_spawn {
3524                        spawn.complete(snapshot);
3525                    }
3526                    Some(child)
3527                }
3528                Err(err) => {
3529                    if runtime.health.critical {
3530                        error!(
3531                            module_id = %spec.module_id,
3532                            program = %spec.program.display(),
3533                            error = %err,
3534                            "critical configured module failed to spawn; marking failed and alerting"
3535                        );
3536                    } else {
3537                        error!(
3538                            module_id = %spec.module_id,
3539                            program = %spec.program.display(),
3540                            error = %err,
3541                            "configured module failed to spawn; marking failed and continuing"
3542                        );
3543                    }
3544                    fail_snapshot(
3545                        snapshot,
3546                        Some(&spec.module_id),
3547                        None,
3548                        &runtime.failure_context(daemon_initiated),
3549                    );
3550                    None
3551                }
3552            }
3553        } else {
3554            None
3555        };
3556        self.start_monitor(spec, runtime, &module, child, commands);
3557        Ok(module)
3558    }
3559
3560    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3561        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3562        SupervisorRuntimeConfig {
3563            registry: self.registry.clone(),
3564            recovery_is_daemon: true,
3565            start_requester: None,
3566            pending_activity_restart: false,
3567            provenance_probe: self.provenance_probe.clone(),
3568            scheduled_respawn: Arc::default(),
3569            deferred_reload_reply: Arc::default(),
3570            restart_policy: self.restart_policy,
3571            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3572            drain_timeout: self.drain_timeout,
3573            // Shared with this module's roster copy: daemon shutdown waits on
3574            // each child for the module's own drain budget, as resolved now.
3575            child_roster: self
3576                .child_roster
3577                .for_module(Arc::clone(&effective_drain_timeout)),
3578            effective_drain_timeout,
3579            default_drain_timeout: self.drain_timeout,
3580            health: self.health.clone(),
3581            connection_file_path: self.connection_file_path.clone(),
3582            capture_logs_dir: self.capture_logs_dir.clone(),
3583            forwarding: self.forwarding.clone(),
3584            supervisor_handle: self.supervisor_handle.clone(),
3585            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3586            terminal_ring: Arc::new(Mutex::new(
3587                TerminalRing::new(
3588                    TerminalRingConfig::default(),
3589                    self.daemon_start_clock.started_at_ms(),
3590                )
3591                .with_start_clock(self.daemon_start_clock)
3592                .with_journal(self.terminal_journal.clone())
3593                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3594            )),
3595            spawn_events: self.spawn_events.clone(),
3596            #[cfg(target_os = "linux")]
3597            cgroup_placement: self.cgroup_placement.clone(),
3598            #[cfg(test)]
3599            test_seed_stale_facts_before_enable_spawn: false,
3600            #[cfg(test)]
3601            test_reload_exit_record_gate: None,
3602        }
3603    }
3604
3605    fn supervised_module(
3606        &self,
3607        spec: ModuleSpec,
3608        runtime: SupervisorRuntimeConfig,
3609        snapshot: SharedSnapshot,
3610        child: Option<SupervisedChild>,
3611    ) -> SupervisedModule {
3612        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3613        self.start_monitor(spec, runtime, &module, child, commands);
3614        module
3615    }
3616
3617    fn register_module(
3618        &self,
3619        spec: &ModuleSpec,
3620        runtime: &SupervisorRuntimeConfig,
3621        snapshot: SharedSnapshot,
3622    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3623        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3624            spec: spec.clone(),
3625            health: runtime.health.clone(),
3626        }));
3627        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3628        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3629        // The module's OWN policy, which may be its per-module config rather than
3630        // the supervisor-wide one; status must report the budget the supervise
3631        // loop actually enforces.
3632        let restart_policy = runtime.restart_policy;
3633        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3634        let (tx, rx) = mpsc::channel(4);
3635        let module_id = spec.module_id.clone();
3636        let module = SupervisedModule {
3637            inner: Arc::new(SupervisedModuleInner {
3638                module_id: module_id.clone(),
3639                registry: Arc::clone(&self.registry),
3640                snapshot,
3641                configuration,
3642                stderr_ring,
3643                terminal_ring,
3644                commands: tx,
3645                monitor: Mutex::new(None),
3646                restart_policy,
3647                effective_drain_timeout,
3648                provenance_probe: self.provenance_probe.clone(),
3649                retained: runtime.child_roster.retained_view(),
3650            }),
3651        };
3652        // Identity gates and the configured mark must precede the roster entry,
3653        // so a client never sees a module without its admission policy.
3654        if let Some(supervisor_handle) = &self.supervisor_handle {
3655            supervisor_handle.insert(module.clone());
3656        }
3657        (module, rx)
3658    }
3659
3660    fn start_monitor(
3661        &self,
3662        spec: ModuleSpec,
3663        runtime: SupervisorRuntimeConfig,
3664        module: &SupervisedModule,
3665        child: Option<SupervisedChild>,
3666        commands: mpsc::Receiver<SupervisorCommand>,
3667    ) {
3668        let monitor = tokio::spawn(supervise_loop(
3669            spec,
3670            runtime,
3671            Arc::clone(&self.registry),
3672            Arc::clone(&self.process_liveness),
3673            Arc::clone(&module.inner.snapshot),
3674            child,
3675            commands,
3676        ));
3677        *module
3678            .inner
3679            .monitor
3680            .lock()
3681            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3682    }
3683}
3684
3685impl Default for Supervisor {
3686    fn default() -> Self {
3687        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3688    }
3689}
3690
3691/// Handle to one supervised child process.
3692#[derive(Clone)]
3693pub struct SupervisedModule {
3694    inner: Arc<SupervisedModuleInner>,
3695}
3696
3697struct SupervisedModuleInner {
3698    retained: crate::child_roster::RetainedView,
3699    module_id: String,
3700    registry: Arc<Registry>,
3701    snapshot: SharedSnapshot,
3702    configuration: Arc<Mutex<SupervisedConfiguration>>,
3703    stderr_ring: Arc<Mutex<StderrRing>>,
3704    terminal_ring: Arc<Mutex<TerminalRing>>,
3705    commands: mpsc::Sender<SupervisorCommand>,
3706    monitor: Mutex<Option<JoinHandle<()>>>,
3707    /// Copied from the supervisor's runtime config at spawn so `status()` can
3708    /// report the restart budget without reaching back into the supervisor. The
3709    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3710    restart_policy: RestartPolicy,
3711    effective_drain_timeout: Arc<Mutex<Duration>>,
3712    provenance_probe: ExecutableIdentityProbe,
3713}
3714
3715impl fmt::Debug for SupervisedModule {
3716    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3717        f.debug_struct("SupervisedModule")
3718            .field("module_id", &self.inner.module_id)
3719            .field("status", &self.status())
3720            .finish_non_exhaustive()
3721    }
3722}
3723
3724impl SupervisedModule {
3725    pub fn module_id(&self) -> &str {
3726        &self.inner.module_id
3727    }
3728
3729    /// Test-only: put one probe miss on the streak, the way
3730    /// `handle_health_probe_failure` does, so tests can assert what a later
3731    /// event does to the streak without driving the whole probe loop.
3732    #[cfg(test)]
3733    pub(crate) fn record_health_probe_failure_for_test(
3734        &self,
3735        detail: &str,
3736    ) -> Result<(), SuperviseError> {
3737        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3738            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3739            state.health.detail = Some(detail.to_string());
3740        })
3741    }
3742
3743    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3744        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3745    }
3746
3747    pub(crate) fn registration_end_reason(
3748        &self,
3749    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3750        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3751        Ok(match snapshot.state {
3752            ModuleState::Draining if snapshot.draining_to_replace => {
3753                Some(RegistrationEndReason::SupervisorRestart)
3754            }
3755            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3756            ModuleState::Restarting if snapshot.draining_to_replace => {
3757                Some(RegistrationEndReason::SupervisorRestart)
3758            }
3759            _ => None,
3760        })
3761    }
3762
3763    /// The module's retained stderr, newest lines last.
3764    ///
3765    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3766    /// module, `supervisor.list` renders every module, and putting it in the
3767    /// shared snapshot would make each status read carry a payload almost nobody
3768    /// asked for. Callers that want the text ask for it.
3769    pub fn stderr_tail(
3770        &self,
3771        max_lines: Option<usize>,
3772        max_bytes: Option<usize>,
3773    ) -> StderrTailSnapshot {
3774        self.inner
3775            .stderr_ring
3776            .lock()
3777            .unwrap_or_else(|poisoned| poisoned.into_inner())
3778            .snapshot(max_lines, max_bytes)
3779    }
3780
3781    /// The module's bounded terminal history, oldest retained exit first.
3782    ///
3783    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3784    /// daemon whose in-memory history was necessarily reset.
3785    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3786        self.inner
3787            .terminal_ring
3788            .lock()
3789            .unwrap_or_else(|poisoned| poisoned.into_inner())
3790            .snapshot()
3791    }
3792
3793    /// Retained observations from the current ring and all journal generations.
3794    ///
3795    /// Blocking: this reads the journal files. Async callers use
3796    /// `Self::read_durable_terminal_history`.
3797    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3798        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3799    }
3800
3801    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3802    /// read (up to every retained generation) never occupies a runtime worker.
3803    /// Fails only if the blocking task could not finish (runtime shutdown or a
3804    /// panic in the read).
3805    pub(crate) async fn read_durable_terminal_history(
3806        &self,
3807    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3808        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3809        let module_id = self.inner.module_id.clone();
3810        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3811            .await
3812    }
3813
3814    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3815        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3816            .map(|(status, _)| status)
3817    }
3818
3819    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3820        let state = lock_snapshot(&self.inner.snapshot)?;
3821        Ok(state
3822            .failed_retry
3823            .pending
3824            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3825            .map(|retry| {
3826                unix_ms_now().saturating_add(
3827                    u64::try_from(
3828                        retry
3829                            .deadline
3830                            .saturating_duration_since(Instant::now())
3831                            .as_millis(),
3832                    )
3833                    .unwrap_or(u64::MAX),
3834                )
3835            }))
3836    }
3837
3838    pub(crate) fn record_deliberate_severance(
3839        &self,
3840        identity: ProcessIdentity,
3841    ) -> Result<bool, SuperviseError> {
3842        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3843        if snapshot.pid != Some(identity.pid)
3844            || snapshot.process_start_time != Some(identity.start_time)
3845        {
3846            return Ok(false);
3847        }
3848        snapshot.deliberate_severance = Some(identity);
3849        Ok(true)
3850    }
3851
3852    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3853    ///
3854    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3855    /// does not produce reader-observability logs.
3856    pub(crate) fn status_for_control(
3857        &self,
3858        caller: &'static str,
3859    ) -> Result<ModuleStatus, SuperviseError> {
3860        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3861            .map(|(status, _)| status)
3862    }
3863
3864    fn status_with_snapshot_lock(
3865        &self,
3866        snapshot: &SharedSnapshot,
3867        caller: Option<&'static str>,
3868    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3869        let mut guard = match caller {
3870            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3871            None => lock_snapshot(snapshot)?,
3872        };
3873        // Read the budget through the pruning path so a reader sees the same
3874        // in-window count the restart decision would use, not a stale total.
3875        let restart_count =
3876            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3877        let snapshot = guard.clone();
3878        drop(guard);
3879        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3880            SuperviseError::StatePoisoned {
3881                module_id: Some(self.inner.module_id.clone()),
3882            }
3883        })?;
3884        let registration_active = self
3885            .inner
3886            .registry
3887            .get_module(&self.inner.module_id)
3888            .map_err(SuperviseError::Registry)?
3889            .is_some();
3890        let protocol = snapshot
3891            .spawned_protocol
3892            .unwrap_or(self.declared_protocol()?);
3893        let running_process =
3894            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3895        // Registration is the difference between the two protocols and the only
3896        // one: a subc module that has not registered cannot serve a request even
3897        // though its process is up, and a `none` module never registers at all,
3898        // so requiring it there would pin `live` to false for the whole life of
3899        // a perfectly healthy process.
3900        let live = match protocol {
3901            ModuleProtocol::Subc => running_process && registration_active,
3902            ModuleProtocol::None => running_process,
3903        };
3904        // Read only the retained cache: a status read must not resend signals,
3905        // rewrite the record, or keep the containment resource alive.
3906        let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3907        let carry_configured = self
3908            .inner
3909            .configuration
3910            .lock()
3911            .map_err(|_| SuperviseError::StatePoisoned {
3912                module_id: Some(self.inner.module_id.clone()),
3913            })?
3914            .spec
3915            .carry_containment;
3916        let recovery_block = carries_containment(
3917            carry_configured,
3918            self.inner
3919                .retained
3920                .has_carried_container(&self.inner.module_id),
3921        )
3922        .then(|| self.inner.retained.recovery_block())
3923        .flatten();
3924        let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3925        let pid = snapshot.reported_pid();
3926        let mut health = snapshot.health;
3927        if let Some(detail) = recovery_block
3928            .as_ref()
3929            .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3930        {
3931            health.detail = Some(match health.detail {
3932                Some(previous) => format!("{previous}; {detail}"),
3933                None => detail.clone(),
3934            });
3935        }
3936
3937        Ok((
3938            ModuleStatus {
3939                waiting_for_previous_pid: retained
3940                    .as_ref()
3941                    .map(|(pid, _)| *pid)
3942                    .filter(|pid| *pid > 0)
3943                    .or(snapshot.waiting_for_previous_pid),
3944                waiting_for_previous_detail: recovery_block
3945                    .or_else(|| retained.map(|(_, detail)| detail))
3946                    .or(snapshot.waiting_for_previous_detail),
3947                module_id: self.inner.module_id.clone(),
3948                launch_nonce_source,
3949                state: snapshot.state,
3950                enabled: snapshot.enabled,
3951                process_alive: snapshot.process_alive,
3952                registration_active,
3953                protocol,
3954                live,
3955                restart_count,
3956                lifetime_restarts: snapshot.lifetime_restarts,
3957                spawn_generation: snapshot.spawn_generation,
3958                max_restarts: self.inner.restart_policy.max_restarts,
3959                restart_window: self.inner.restart_policy.window,
3960                drain_timeout,
3961                restart_backoff: self.inner.restart_policy.backoff,
3962                restart_max_backoff: self.inner.restart_policy.max_backoff,
3963                pid,
3964                spawned_at_ms: snapshot.spawned_at_ms,
3965                spawned_from: snapshot.spawned_from,
3966                launch_spec_digest: snapshot.launch_spec_digest,
3967                process_start_time: snapshot.process_start_time,
3968                last_exit: snapshot.last_exit,
3969                health,
3970            },
3971            snapshot.spawned_file_identity.clone(),
3972        ))
3973    }
3974
3975    #[cfg(test)]
3976    pub(crate) fn hold_snapshot_for_test(
3977        &self,
3978        acquired: std::sync::mpsc::Sender<()>,
3979        hold: Duration,
3980    ) -> std::thread::JoinHandle<()> {
3981        let snapshot = Arc::clone(&self.inner.snapshot);
3982        std::thread::spawn(move || {
3983            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3984            acquired
3985                .send(())
3986                .expect("test receiver waits for snapshot lock");
3987            std::thread::sleep(hold);
3988        })
3989    }
3990
3991    /// The status and the running-image check for `supervisor.provenance`,
3992    /// taken from one status read. The exec acknowledgement can land between
3993    /// two separate reads, and the reply would then pair "no pid yet" with an
3994    /// image observed after the module started, which describes no single
3995    /// moment.
3996    pub(crate) async fn status_and_running_image_agreement(
3997        &self,
3998    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3999        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
4000        let image = self
4001            .inner
4002            .provenance_probe
4003            .observe(
4004                status.pid,
4005                status.spawned_from.as_deref(),
4006                identity,
4007                status.process_start_time,
4008            )
4009            .await;
4010        Ok((status, image))
4011    }
4012
4013    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4014        let snapshot = match lock_snapshot(&self.inner.snapshot) {
4015            Ok(snapshot) => snapshot.clone(),
4016            Err(_) => {
4017                return subc_control::RunningImageAgreement::Unavailable {
4018                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
4019                };
4020            }
4021        };
4022        self.inner
4023            .provenance_probe
4024            .observe(
4025                snapshot.reported_pid(),
4026                snapshot.spawned_from.as_deref(),
4027                snapshot.spawned_file_identity.clone(),
4028                snapshot.process_start_time,
4029            )
4030            .await
4031    }
4032
4033    /// Memory and CPU time of the module's current process, read now. Only the
4034    /// process the supervisor spawned is read, not processes it has started.
4035    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4036        #[cfg(windows)]
4037        {
4038            let image = lock_snapshot(&self.inner.snapshot)
4039                .ok()
4040                .and_then(|snapshot| {
4041                    snapshot
4042                        .spawned_file_identity
4043                        .as_ref()
4044                        .and_then(|identity| identity.windows_image.clone())
4045                });
4046            match image {
4047                Some(image) => {
4048                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
4049                }
4050                None => subc_control::ChildResourceUsage::Unavailable {
4051                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4052                },
4053            }
4054        }
4055        #[cfg(not(windows))]
4056        {
4057            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4058                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4059                Err(_) => {
4060                    return subc_control::ChildResourceUsage::Unavailable {
4061                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4062                    }
4063                }
4064            };
4065            crate::child_resources::read(pid, start_time)
4066        }
4067    }
4068
4069    #[cfg(all(test, any(target_os = "linux", windows)))]
4070    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4071        self.inner
4072            .snapshot
4073            .try_lock()
4074            .map(|snapshot| snapshot.subtree_target.clone())
4075            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4076    }
4077
4078    #[cfg(test)]
4079    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4080        self.inner.snapshot.lock().unwrap().subtree_target = target;
4081    }
4082
4083    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4084        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4085        Ok(match snapshot.state {
4086            ModuleState::Restarting => true,
4087            ModuleState::Failed | ModuleState::Disabled => false,
4088            _ if snapshot.enabled && snapshot.respawn_pending => true,
4089            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4090        })
4091    }
4092
4093    #[cfg(test)]
4094    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4095        self.is_warming_with_snapshot_lock(None)
4096    }
4097
4098    pub(crate) fn is_warming_for_control(
4099        &self,
4100        caller: &'static str,
4101    ) -> Result<bool, SuperviseError> {
4102        self.is_warming_with_snapshot_lock(Some(caller))
4103    }
4104
4105    fn is_warming_with_snapshot_lock(
4106        &self,
4107        caller: Option<&'static str>,
4108    ) -> Result<bool, SuperviseError> {
4109        let snapshot = match caller {
4110            Some(caller) => {
4111                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4112            }
4113            None => lock_snapshot(&self.inner.snapshot)?,
4114        }
4115        .clone();
4116        Ok(matches!(
4117            snapshot.state,
4118            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4119        ))
4120    }
4121
4122    /// Drain the module and stop monitoring it.
4123    pub async fn drain(&self) -> Result<(), SuperviseError> {
4124        self.stop().await
4125    }
4126
4127    pub(crate) async fn retire_requested(
4128        &self,
4129        requester: Option<Principal>,
4130    ) -> Result<(), SuperviseError> {
4131        match self.state()? {
4132            ModuleState::Stopped => return Ok(()),
4133            ModuleState::Failed
4134                if lock_snapshot(&self.inner.snapshot)?
4135                    .failed_retry
4136                    .pending
4137                    .is_none() =>
4138            {
4139                return Ok(())
4140            }
4141            ModuleState::Failed
4142            | ModuleState::Starting
4143            | ModuleState::Running
4144            | ModuleState::Unresponsive
4145            | ModuleState::Restarting
4146            | ModuleState::Draining
4147            | ModuleState::Disabled => {}
4148        }
4149
4150        let (reply_tx, reply_rx) = oneshot::channel();
4151        self.inner
4152            .commands
4153            .send(SupervisorCommand::Retire {
4154                requester,
4155                reply: reply_tx,
4156            })
4157            .await
4158            .map_err(|_| SuperviseError::CommandClosed {
4159                module_id: self.inner.module_id.clone(),
4160            })?;
4161        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4162            module_id: self.inner.module_id.clone(),
4163        })?
4164    }
4165
4166    pub async fn stop(&self) -> Result<(), SuperviseError> {
4167        self.stop_requested(None).await
4168    }
4169
4170    pub(crate) async fn stop_requested(
4171        &self,
4172        requester: Option<Principal>,
4173    ) -> Result<(), SuperviseError> {
4174        match self.state()? {
4175            ModuleState::Stopped => return Ok(()),
4176            ModuleState::Failed
4177                if lock_snapshot(&self.inner.snapshot)?
4178                    .failed_retry
4179                    .pending
4180                    .is_none() =>
4181            {
4182                return Ok(())
4183            }
4184            ModuleState::Failed
4185            | ModuleState::Starting
4186            | ModuleState::Running
4187            | ModuleState::Unresponsive
4188            | ModuleState::Restarting
4189            | ModuleState::Draining
4190            | ModuleState::Disabled => {}
4191        }
4192
4193        let (reply_tx, reply_rx) = oneshot::channel();
4194        self.inner
4195            .commands
4196            .send(SupervisorCommand::Drain {
4197                requester,
4198                reply: reply_tx,
4199            })
4200            .await
4201            .map_err(|_| SuperviseError::CommandClosed {
4202                module_id: self.inner.module_id.clone(),
4203            })?;
4204        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4205            module_id: self.inner.module_id.clone(),
4206        })?
4207    }
4208
4209    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4210        self.restart_requested(drain_timeout_ms, None).await
4211    }
4212
4213    pub(crate) async fn restart_requested(
4214        &self,
4215        drain_timeout_ms: Option<u64>,
4216        requester: Option<Principal>,
4217    ) -> Result<(), SuperviseError> {
4218        let queued_at = Instant::now();
4219        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4220        let (reply_tx, reply_rx) = oneshot::channel();
4221        self.inner
4222            .commands
4223            .send(SupervisorCommand::Restart {
4224                requester,
4225                drain_timeout_ms,
4226                received_at_generation,
4227                queued_at,
4228                reply: reply_tx,
4229            })
4230            .await
4231            .map_err(|_| SuperviseError::CommandClosed {
4232                module_id: self.inner.module_id.clone(),
4233            })?;
4234        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4235            module_id: self.inner.module_id.clone(),
4236        })?
4237    }
4238
4239    /// Blue/green restart: see `SupervisorCommand::Swap` and the
4240    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4241    /// process then drains in the background of the supervise loop) or has
4242    /// failed, leaving the old process serving.
4243    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4244        self.swap_requested(ready_timeout, None).await
4245    }
4246
4247    pub(crate) async fn swap_requested(
4248        &self,
4249        ready_timeout: Option<Duration>,
4250        requester: Option<Principal>,
4251    ) -> Result<(), SuperviseError> {
4252        let (reply_tx, reply_rx) = oneshot::channel();
4253        self.inner
4254            .commands
4255            .send(SupervisorCommand::Swap {
4256                requester,
4257                ready_timeout,
4258                reply: reply_tx,
4259            })
4260            .await
4261            .map_err(|_| SuperviseError::CommandClosed {
4262                module_id: self.inner.module_id.clone(),
4263            })?;
4264        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4265            module_id: self.inner.module_id.clone(),
4266        })?
4267    }
4268
4269    pub async fn reload(&self) -> Result<(), SuperviseError> {
4270        self.reload_requested(None).await
4271    }
4272
4273    pub(crate) async fn reload_requested(
4274        &self,
4275        requester: Option<Principal>,
4276    ) -> Result<(), SuperviseError> {
4277        let (reply_tx, reply_rx) = oneshot::channel();
4278        self.inner
4279            .commands
4280            .send(SupervisorCommand::Reload {
4281                requester,
4282                reply: reply_tx,
4283            })
4284            .await
4285            .map_err(|_| SuperviseError::CommandClosed {
4286                module_id: self.inner.module_id.clone(),
4287            })?;
4288        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4289            module_id: self.inner.module_id.clone(),
4290        })?
4291    }
4292
4293    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4294        self.set_enabled_requested(enabled, None).await
4295    }
4296
4297    pub(crate) async fn set_enabled_requested(
4298        &self,
4299        enabled: bool,
4300        requester: Option<Principal>,
4301    ) -> Result<bool, SuperviseError> {
4302        let (reply_tx, reply_rx) = oneshot::channel();
4303        self.inner
4304            .commands
4305            .send(SupervisorCommand::SetEnabled {
4306                requester,
4307                enabled,
4308                reply: reply_tx,
4309            })
4310            .await
4311            .map_err(|_| SuperviseError::CommandClosed {
4312                module_id: self.inner.module_id.clone(),
4313            })?;
4314        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4315            module_id: self.inner.module_id.clone(),
4316        })?
4317    }
4318
4319    /// The current process's protocol, or the configured protocol when down.
4320    /// A rescan stores the next launch spec without changing how an existing
4321    /// process registers, serves routes, is probed, or exits.
4322    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4323        let configured = self
4324            .inner
4325            .configuration
4326            .lock()
4327            .map_err(|_| SuperviseError::StatePoisoned {
4328                module_id: Some(self.inner.module_id.clone()),
4329            })?
4330            .spec
4331            .protocol;
4332        let state = lock_snapshot(&self.inner.snapshot)?;
4333        Ok(state.spawned_protocol.unwrap_or(configured))
4334    }
4335
4336    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4337        let configuration =
4338            self.inner
4339                .configuration
4340                .lock()
4341                .map_err(|_| SuperviseError::StatePoisoned {
4342                    module_id: Some(self.inner.module_id.clone()),
4343                })?;
4344        Ok((configuration.spec.clone(), configuration.health.clone()))
4345    }
4346
4347    /// Replace this module's launch spec, keeping its health and drain policy,
4348    /// the way a rescan does for a changed config entry. The running process is
4349    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4350    #[cfg(any(test, feature = "test-support"))]
4351    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4352        let (_, health) = self.configuration()?;
4353        let drain_timeout_ms = u64::try_from(
4354            self.inner
4355                .effective_drain_timeout
4356                .lock()
4357                .unwrap_or_else(|poisoned| poisoned.into_inner())
4358                .as_millis(),
4359        )
4360        .ok();
4361        self.update_configuration(spec, health, drain_timeout_ms)
4362            .await
4363    }
4364
4365    pub(crate) async fn update_configuration(
4366        &self,
4367        spec: ModuleSpec,
4368        health: HealthConfig,
4369        drain_timeout_ms: Option<u64>,
4370    ) -> Result<(), SuperviseError> {
4371        if spec.module_id != self.inner.module_id {
4372            return Err(SuperviseError::InvalidSpec {
4373                reason: "a supervised module's module_id cannot be changed".to_string(),
4374            });
4375        }
4376        validate_spec(&spec)?;
4377        let (reply_tx, reply_rx) = oneshot::channel();
4378        self.inner
4379            .commands
4380            .send(SupervisorCommand::UpdateConfiguration {
4381                spec: spec.clone(),
4382                health: health.clone(),
4383                drain_timeout_ms,
4384                reply: reply_tx,
4385            })
4386            .await
4387            .map_err(|_| SuperviseError::CommandClosed {
4388                module_id: self.inner.module_id.clone(),
4389            })?;
4390        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4391            module_id: self.inner.module_id.clone(),
4392        })?;
4393        let mut configuration =
4394            self.inner
4395                .configuration
4396                .lock()
4397                .map_err(|_| SuperviseError::StatePoisoned {
4398                    module_id: Some(self.inner.module_id.clone()),
4399                })?;
4400        configuration.spec = spec;
4401        configuration.health = health;
4402        Ok(())
4403    }
4404}
4405
4406impl Drop for SupervisedModuleInner {
4407    fn drop(&mut self) {
4408        let Ok(mut monitor) = self.monitor.lock() else {
4409            return;
4410        };
4411        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4412            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4413                state.state = ModuleState::Stopped;
4414                clear_current_process_facts(state);
4415            });
4416            monitor.abort();
4417        }
4418        let _ = monitor.take();
4419    }
4420}
4421
4422#[derive(Debug)]
4423enum SupervisorCommand {
4424    Drain {
4425        requester: Option<Principal>,
4426        reply: oneshot::Sender<Result<(), SuperviseError>>,
4427    },
4428    Retire {
4429        requester: Option<Principal>,
4430        reply: oneshot::Sender<Result<(), SuperviseError>>,
4431    },
4432    Restart {
4433        requester: Option<Principal>,
4434        /// Operator override for this one restart's drain budget, in ms. `None`
4435        /// uses the module's configured/default budget; `Some(0)` cuts
4436        /// immediately (wedge bounce: a stuck request never settles, so
4437        /// waiting only delays recovery).
4438        drain_timeout_ms: Option<u64>,
4439        /// The module's `spawn_generation` when the request was received, before
4440        /// it waited in the command queue. A queued restart whose module has
4441        /// since spawned a newer process is already satisfied (see the handler).
4442        received_at_generation: u64,
4443        /// When the request entered the command queue, so the handler can log
4444        /// how long it waited behind the loop's other work.
4445        queued_at: Instant,
4446        reply: oneshot::Sender<Result<(), SuperviseError>>,
4447    },
4448    Reload {
4449        requester: Option<Principal>,
4450        reply: oneshot::Sender<Result<(), SuperviseError>>,
4451    },
4452    SetEnabled {
4453        requester: Option<Principal>,
4454        enabled: bool,
4455        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4456    },
4457    UpdateConfiguration {
4458        spec: ModuleSpec,
4459        health: HealthConfig,
4460        /// Per-module drain override from the new config; `None` re-resolves to
4461        /// the supervisor-wide default.
4462        drain_timeout_ms: Option<u64>,
4463        reply: oneshot::Sender<()>,
4464    },
4465    Swap {
4466        requester: Option<Principal>,
4467        /// How long the candidate may take to register and declare itself
4468        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4469        ready_timeout: Option<Duration>,
4470        /// Answered at cutover or failure; the incumbent's drain follows.
4471        reply: oneshot::Sender<Result<(), SuperviseError>>,
4472    },
4473}
4474
4475#[derive(Debug)]
4476pub enum SuperviseError {
4477    InvalidSpec {
4478        reason: String,
4479    },
4480    Spawn {
4481        program: PathBuf,
4482        source: io::Error,
4483        cgroup_path: Option<PathBuf>,
4484    },
4485    Cgroup {
4486        module_id: String,
4487        source: io::Error,
4488    },
4489    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4490    /// than spawn a reserved module without its identity binding.
4491    LaunchNonce {
4492        reason: String,
4493    },
4494    Wait {
4495        module_id: String,
4496        source: io::Error,
4497    },
4498    Kill {
4499        module_id: String,
4500        source: io::Error,
4501    },
4502    Forwarding(ForwardingError),
4503    Registry(RegistryError),
4504    ReloadUnavailable {
4505        module_id: String,
4506        reason: String,
4507    },
4508    /// An operator restart/reload was requested for a module that is currently
4509    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4510    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4511    /// by a restart, so these commands are rejected instead of re-enabling it.
4512    Disabled {
4513        module_id: String,
4514    },
4515    ReloadFailed {
4516        module_id: String,
4517        reason: String,
4518    },
4519    RegistrationStillActive {
4520        module_id: String,
4521        waited: Duration,
4522    },
4523    StatePoisoned {
4524        module_id: Option<String>,
4525    },
4526    CommandClosed {
4527        module_id: String,
4528    },
4529    /// A restart or reload arrived while a swap's candidate was warming. The
4530    /// swap owns the module until it cuts over or fails; a stop or disable
4531    /// would have aborted it instead.
4532    SwapInProgress {
4533        module_id: String,
4534    },
4535    /// A swap was refused before anything was spawned.
4536    SwapRefused {
4537        module_id: String,
4538        reason: SwapRefusal,
4539    },
4540    /// A swap spawned a candidate and gave up on it. The candidate has been
4541    /// killed and its slot freed; the incumbent was left serving and was never
4542    /// drained, except in the one `CutoverLost` case described on that arm.
4543    SwapFailed {
4544        module_id: String,
4545        arm: SwapFailureArm,
4546        detail: String,
4547        /// How the candidate exited, when it exited on its own before the
4548        /// supervisor gave up on it.
4549        candidate_exit: Option<ExitReport>,
4550    },
4551}
4552
4553/// Why a swap was refused before a candidate was spawned.
4554#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4555pub enum SwapRefusal {
4556    /// The module's config does not declare `overlap: "safe"`.
4557    OverlapExclusive,
4558    /// The module is not registered, so there is no incumbent to keep serving
4559    /// and nothing a swap would improve on; a plain restart is the tool.
4560    NotRegistered,
4561    /// The module does not speak the subc wire, so a candidate could never
4562    /// register or declare itself ready.
4563    ProtocolNone,
4564    /// The supervisor lacks the forwarding table (to cut routes over) or the
4565    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4566    NotConfigured,
4567    /// A swap is already open for this module.
4568    AlreadySwapping,
4569}
4570
4571impl SwapRefusal {
4572    pub fn as_str(self) -> &'static str {
4573        match self {
4574            Self::OverlapExclusive => "overlap_exclusive",
4575            Self::NotRegistered => "not_registered",
4576            Self::ProtocolNone => "protocol_none",
4577            Self::NotConfigured => "not_configured",
4578            Self::AlreadySwapping => "already_swapping",
4579        }
4580    }
4581}
4582
4583/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4584/// serving and undrained; see `CutoverLost`.
4585#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4586pub enum SwapFailureArm {
4587    /// The candidate process could not be started.
4588    SpawnFailed,
4589    /// The candidate did not register within the readiness budget.
4590    NeverRegistered,
4591    /// The candidate registered but did not declare itself ready in time.
4592    NeverReady,
4593    /// The candidate exited before cutover.
4594    CandidateExited,
4595    /// The candidate declared itself ready but failed its health probe.
4596    CandidateUnhealthy,
4597    /// An operator stop, disable or retire arrived while the candidate warmed.
4598    /// The candidate was killed and the operator's command then carried out on
4599    /// the incumbent.
4600    Interrupted,
4601    /// The candidate's connection closed at the moment of cutover. If it
4602    /// closed before forwarding moved, the incumbent is untouched. If it closed
4603    /// between the forwarding and registry halves of cutover, forwarding can no
4604    /// longer route to the incumbent, so the module is restarted plainly.
4605    CutoverLost,
4606}
4607
4608impl SwapFailureArm {
4609    pub fn as_str(self) -> &'static str {
4610        match self {
4611            Self::SpawnFailed => "spawn_failed",
4612            Self::NeverRegistered => "never_registered",
4613            Self::NeverReady => "never_ready",
4614            Self::CandidateExited => "candidate_exited",
4615            Self::CandidateUnhealthy => "candidate_unhealthy",
4616            Self::Interrupted => "interrupted",
4617            Self::CutoverLost => "cutover_lost",
4618        }
4619    }
4620}
4621
4622impl fmt::Display for SuperviseError {
4623    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4624        match self {
4625            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4626            Self::Spawn {
4627                program,
4628                source,
4629                cgroup_path: Some(cgroup_path),
4630            } => write!(
4631                f,
4632                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4633                cgroup_path.display(),
4634                program.display()
4635            ),
4636            Self::Spawn {
4637                program,
4638                source,
4639                cgroup_path: None,
4640            } => write!(
4641                f,
4642                "failed to spawn module '{}': {source}",
4643                program.display()
4644            ),
4645            Self::Cgroup { module_id, source } => {
4646                write!(
4647                    f,
4648                    "failed to prepare cgroup for module '{module_id}': {source}"
4649                )
4650            }
4651            Self::LaunchNonce { reason } => {
4652                write!(
4653                    f,
4654                    "failed to generate reserved-module launch nonce: {reason}"
4655                )
4656            }
4657            Self::Wait { module_id, source } => {
4658                write!(f, "failed to wait for module '{module_id}': {source}")
4659            }
4660            Self::Kill { module_id, source } => {
4661                write!(f, "failed to kill module '{module_id}': {source}")
4662            }
4663            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4664            Self::Registry(err) => write!(f, "registry error: {err}"),
4665            Self::ReloadUnavailable { module_id, reason } => {
4666                write!(f, "reload unavailable for module '{module_id}': {reason}")
4667            }
4668            Self::Disabled { module_id } => {
4669                write!(
4670                    f,
4671                    "module '{module_id}' is disabled; enable it before restart or reload"
4672                )
4673            }
4674            Self::ReloadFailed { module_id, reason } => {
4675                write!(f, "reload failed for module '{module_id}': {reason}")
4676            }
4677            Self::RegistrationStillActive { module_id, waited } => write!(
4678                f,
4679                "module '{module_id}' registration remained active after waiting {waited:?}"
4680            ),
4681            Self::StatePoisoned { module_id } => match module_id {
4682                Some(module_id) => {
4683                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4684                }
4685                None => write!(f, "supervisor state was poisoned"),
4686            },
4687            Self::CommandClosed { module_id } => {
4688                write!(
4689                    f,
4690                    "supervisor command channel for module '{module_id}' is closed"
4691                )
4692            }
4693            Self::SwapInProgress { module_id } => write!(
4694                f,
4695                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4696            ),
4697            Self::SwapRefused { module_id, reason } => match reason {
4698                SwapRefusal::OverlapExclusive => write!(
4699                    f,
4700                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4701                ),
4702                SwapRefusal::NotRegistered => write!(
4703                    f,
4704                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4705                ),
4706                SwapRefusal::ProtocolNone => write!(
4707                    f,
4708                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4709                ),
4710                SwapRefusal::NotConfigured => write!(
4711                    f,
4712                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4713                ),
4714                SwapRefusal::AlreadySwapping => {
4715                    write!(f, "module '{module_id}' is already being swapped")
4716                }
4717            },
4718            Self::SwapFailed {
4719                module_id,
4720                arm,
4721                detail,
4722                ..
4723            } => write!(
4724                f,
4725                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4726                arm.as_str()
4727            ),
4728        }
4729    }
4730}
4731
4732impl Error for SuperviseError {
4733    fn source(&self) -> Option<&(dyn Error + 'static)> {
4734        match self {
4735            Self::Spawn { source, .. }
4736            | Self::Cgroup { source, .. }
4737            | Self::Wait { source, .. }
4738            | Self::Kill { source, .. } => Some(source),
4739            Self::Forwarding(err) => Some(err),
4740            Self::Registry(err) => Some(err),
4741            Self::LaunchNonce { .. }
4742            | Self::InvalidSpec { .. }
4743            | Self::ReloadUnavailable { .. }
4744            | Self::Disabled { .. }
4745            | Self::ReloadFailed { .. }
4746            | Self::RegistrationStillActive { .. }
4747            | Self::StatePoisoned { .. }
4748            | Self::CommandClosed { .. }
4749            | Self::SwapInProgress { .. }
4750            | Self::SwapRefused { .. }
4751            | Self::SwapFailed { .. } => None,
4752        }
4753    }
4754}
4755
4756pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4757    if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4758        return Err(SuperviseError::InvalidSpec {
4759            reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4760        });
4761    }
4762    if spec.module_id.trim().is_empty() {
4763        return Err(SuperviseError::InvalidSpec {
4764            reason: "module_id must not be empty".to_string(),
4765        });
4766    }
4767
4768    Ok(())
4769}
4770
4771#[derive(Debug, Default)]
4772struct HealthProbeRuntime {
4773    spawn_generation: Option<u64>,
4774    configured_health: Option<HealthConfig>,
4775    registered_connection: Option<crate::ConnectionId>,
4776    advertised: bool,
4777    next_probe_at: Option<Instant>,
4778    probe_index: u64,
4779}
4780
4781fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4782    lock_snapshot(snapshot)
4783        .ok()
4784        .and_then(|state| state.spawned_protocol)
4785        .unwrap_or(spec.protocol)
4786}
4787
4788impl HealthProbeRuntime {
4789    fn refresh_registration(
4790        &mut self,
4791        spec: &ModuleSpec,
4792        runtime: &SupervisorRuntimeConfig,
4793        registry: &Registry,
4794        snapshot: &SharedSnapshot,
4795    ) {
4796        let generation = lock_snapshot(snapshot)
4797            .ok()
4798            .map(|state| state.spawn_generation);
4799        if self.spawn_generation != generation {
4800            // A replacement cannot reuse a deadline that is already due for its
4801            // predecessor. HTTP children get a full cadence to start; wire
4802            // children still wait for their own health-advertising HELLO.
4803            *self = Self {
4804                spawn_generation: generation,
4805                ..Self::default()
4806            };
4807        }
4808        if self.configured_health.as_ref() != Some(&runtime.health) {
4809            self.configured_health = Some(runtime.health.clone());
4810            self.next_probe_at = None;
4811            self.registered_connection = None;
4812            self.probe_index = 0;
4813        }
4814        // A non-wire process never registers. Only an explicitly configured
4815        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4816        // health failure for that kind of process.
4817        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4818            self.registered_connection = None;
4819            self.advertised = runtime.health.http.is_some();
4820            if !self.advertised {
4821                self.next_probe_at = None;
4822                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4823                    let unknown = ModuleHealthStatus::default();
4824                    if state.health != unknown {
4825                        state.health = unknown;
4826                    }
4827                });
4828            } else if self.next_probe_at.is_none() {
4829                self.next_probe_at = Some(
4830                    Instant::now()
4831                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4832                );
4833            }
4834            return;
4835        }
4836
4837        let registration = match registry.get_module(&spec.module_id) {
4838            Ok(registration) => registration,
4839            Err(err) => {
4840                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4841                self.advertised = false;
4842                self.next_probe_at = None;
4843                return;
4844            }
4845        };
4846
4847        let Some(registration) = registration else {
4848            self.registered_connection = None;
4849            self.advertised = false;
4850            self.next_probe_at = None;
4851            return;
4852        };
4853
4854        let advertised = registration
4855            .control_ops
4856            .iter()
4857            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4858        if !advertised {
4859            self.registered_connection = Some(registration.connection_id);
4860            self.advertised = false;
4861            self.next_probe_at = None;
4862            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4863                if state.health.status != SupervisorHealthStatus::Unknown
4864                    || state.health.consecutive_failures != 0
4865                    || state.health.last_probe_ms.is_some()
4866                    || state.health.detail.is_some()
4867                    || state.health.metrics.is_some()
4868                {
4869                    state.health.status = SupervisorHealthStatus::Unknown;
4870                    state.health.consecutive_failures = 0;
4871                    state.health.last_probe_ms = None;
4872                    state.health.detail = None;
4873                    state.health.metrics = None;
4874                }
4875            });
4876            return;
4877        }
4878
4879        let reregistered = self.registered_connection != Some(registration.connection_id);
4880        self.registered_connection = Some(registration.connection_id);
4881        self.advertised = true;
4882        if reregistered || self.next_probe_at.is_none() {
4883            self.probe_index = 0;
4884            self.next_probe_at = Some(
4885                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4886            );
4887            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4888                state.health.status = SupervisorHealthStatus::Unknown;
4889                state.health.consecutive_failures = 0;
4890                state.health.detail = None;
4891                state.health.metrics = None;
4892            });
4893        }
4894    }
4895
4896    fn wake_at(&self) -> Option<Instant> {
4897        if !self.advertised {
4898            return None;
4899        }
4900        self.next_probe_at
4901    }
4902
4903    fn due(&self) -> bool {
4904        self.advertised
4905            && self
4906                .next_probe_at
4907                .is_some_and(|next| Instant::now() >= next)
4908    }
4909
4910    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4911        self.probe_index = self.probe_index.wrapping_add(1);
4912        self.next_probe_at = Some(
4913            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4914        );
4915    }
4916}
4917
4918/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4919///
4920/// This was a struct with a single `message: String`, and every one of the
4921/// fifteen construction sites collapsed into it. Each site knows exactly what it
4922/// saw -- the lane is gone, the module did not answer in time, the module
4923/// answered with the wrong thing -- and `handle_health_probe_failure` then
4924/// treated all of them identically: increment a counter, compare to a threshold,
4925/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4926/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4927///
4928/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4929///
4930/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4931///   answer on it again.
4932/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4933///   AND with a perfectly healthy one that lost a CPU race -- which is what
4934///   happens under machine load, and is how this supervisor killed a healthy
4935///   module three times in one day.
4936/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4937///   Restarting on it is defensible, but it is not the silence case and should
4938///   never be counted as one.
4939/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4940///   anything, so it cannot be evidence about the module at all.
4941///
4942/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4943/// one that fires most often, and while every variant collapsed into one string
4944/// it carried the same weight as the strongest.
4945///
4946/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4947/// DESIGN and a reader stopping at it gets the build backwards: the restart
4948/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4949/// probes still increment the failure streak and drive escalation at the
4950/// threshold (see `is_proof_of_death` below for why that is deliberate and
4951/// what gates the change). Absence of evidence restarts modules today.
4952#[derive(Debug)]
4953enum HealthProbeEvidence {
4954    /// The module's control lane is gone. Proof of death.
4955    LaneDead,
4956    /// No reply within the deadline. Proves nothing about the module's state.
4957    NoAnswer,
4958    /// The module replied, but not with a usable health report. Proves it is alive.
4959    BadAnswer,
4960    /// The daemon could not ask. Says nothing about the module.
4961    Misconfigured,
4962}
4963
4964#[derive(Debug)]
4965struct HealthProbeError {
4966    evidence: HealthProbeEvidence,
4967    message: String,
4968}
4969
4970impl HealthProbeError {
4971    fn lane_dead(message: impl Into<String>) -> Self {
4972        Self::with(HealthProbeEvidence::LaneDead, message)
4973    }
4974
4975    fn no_answer(message: impl Into<String>) -> Self {
4976        Self::with(HealthProbeEvidence::NoAnswer, message)
4977    }
4978
4979    fn bad_answer(message: impl Into<String>) -> Self {
4980        Self::with(HealthProbeEvidence::BadAnswer, message)
4981    }
4982
4983    fn misconfigured(message: impl Into<String>) -> Self {
4984        Self::with(HealthProbeEvidence::Misconfigured, message)
4985    }
4986
4987    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4988        Self {
4989            evidence,
4990            message: message.into(),
4991        }
4992    }
4993
4994    /// Whether this observation is proof the module cannot serve.
4995    ///
4996    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4997    /// variant that fires under CPU starvation, and treating it as proof is the
4998    /// defect this enum exists to make impossible to reintroduce silently.
4999    ///
5000    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
5001    /// to restart also needs a bound for the case it excludes -- a genuinely
5002    /// wedged module, alive but never answering -- and that bound must come from
5003    /// the distribution of real late-answer latencies, which nothing measures
5004    /// yet. Landing the classification first makes the later change a one-line
5005    /// decision against evidence that already exists, rather than two unproven
5006    /// changes at once.
5007    #[allow(dead_code)]
5008    fn is_proof_of_death(&self) -> bool {
5009        matches!(self.evidence, HealthProbeEvidence::LaneDead)
5010    }
5011
5012    /// Short stable label for logs and the health snapshot.
5013    ///
5014    /// An operator reading `ck health` currently cannot tell "the module is gone"
5015    /// from "the module did not answer in five seconds", because both render as
5016    /// prose in the same field. These labels are what make the two
5017    /// distinguishable at a glance, and they are what a later restart-policy
5018    /// change will be argued from.
5019    fn label(&self) -> &'static str {
5020        match self.evidence {
5021            HealthProbeEvidence::LaneDead => "lane-dead",
5022            HealthProbeEvidence::NoAnswer => "no-answer",
5023            HealthProbeEvidence::BadAnswer => "bad-answer",
5024            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5025        }
5026    }
5027}
5028
5029impl fmt::Display for HealthProbeError {
5030    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5031        f.write_str(&self.message)
5032    }
5033}
5034
5035async fn run_health_probe_cycle(
5036    spec: &ModuleSpec,
5037    runtime: &SupervisorRuntimeConfig,
5038    registry: &Registry,
5039    process_liveness: &SupervisorProcessLiveness,
5040    snapshot: &SharedSnapshot,
5041    child: &mut Option<SupervisedChild>,
5042) {
5043    let now_ms = unix_ms_now();
5044    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5045        .then_some(runtime.health.http.as_deref())
5046        .flatten();
5047    let result = match http {
5048        Some(url) => probe_http_health(url, runtime.health.deadline).await,
5049        None => probe_module_health(&spec.module_id, runtime, None).await,
5050    };
5051    match result {
5052        Ok(report) => {
5053            handle_health_report(
5054                spec,
5055                runtime,
5056                registry,
5057                process_liveness,
5058                snapshot,
5059                child,
5060                report,
5061                now_ms,
5062            )
5063            .await;
5064        }
5065        Err(err) => {
5066            if http.is_some() {
5067                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5068                    state.health.status = SupervisorHealthStatus::Failing;
5069                });
5070            }
5071            handle_health_probe_failure(
5072                spec,
5073                runtime,
5074                registry,
5075                process_liveness,
5076                snapshot,
5077                child,
5078                err,
5079                now_ms,
5080            )
5081            .await;
5082        }
5083    }
5084}
5085
5086pub(crate) struct HttpProbeTarget<'a> {
5087    address: std::net::SocketAddr,
5088    localhost: bool,
5089    authority: &'a str,
5090    path: String,
5091}
5092
5093/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
5094/// or TLS. A URL cannot turn a local health check into an outbound connection.
5095pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5096    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5097        return Err("must not contain whitespace, controls, or a fragment".into());
5098    }
5099    let rest = url
5100        .strip_prefix("http://")
5101        .ok_or("must use plain http://")?;
5102    let split = rest.find(['/', '?']).unwrap_or(rest.len());
5103    let (authority, suffix) = rest.split_at(split);
5104    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5105        ("::1", rest)
5106    } else {
5107        let split = authority.find(':').unwrap_or(authority.len());
5108        authority.split_at(split)
5109    };
5110    let ip: std::net::IpAddr = match host {
5111        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5112        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5113        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5114    };
5115    let port = if port.is_empty() {
5116        80
5117    } else {
5118        port.strip_prefix(':')
5119            .and_then(|p| p.parse::<u16>().ok())
5120            .filter(|p| *p > 0)
5121            .ok_or("must have a valid nonzero TCP port")?
5122    };
5123    let path = if suffix.is_empty() {
5124        "/".into()
5125    } else if suffix.starts_with('?') {
5126        format!("/{suffix}")
5127    } else {
5128        suffix.into()
5129    };
5130    Ok(HttpProbeTarget {
5131        address: std::net::SocketAddr::new(ip, port),
5132        localhost: host == "localhost",
5133        authority,
5134        path,
5135    })
5136}
5137
5138async fn probe_http_health(
5139    url: &str,
5140    deadline: Duration,
5141) -> Result<HealthReport, HealthProbeError> {
5142    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5143    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5144    // Keep partial diagnostics outside the timed future so cancellation does
5145    // not discard a status line or body bytes already received.
5146    let mut response_status = String::new();
5147    let mut body = Vec::new();
5148    let probe = async {
5149        // Resolve localhost ourselves so a hosts-file override cannot turn
5150        // this into an outbound request, while IPv6-only local servers work.
5151        let connection = match tokio::net::TcpStream::connect(target.address).await {
5152            Err(_) if target.localhost => {
5153                tokio::net::TcpStream::connect((
5154                    std::net::Ipv6Addr::LOCALHOST,
5155                    target.address.port(),
5156                ))
5157                .await
5158            }
5159            result => result,
5160        };
5161        let mut stream = connection.map_err(|error| {
5162            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5163        })?;
5164        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5165            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5166        let mut reader = BufReader::new(stream);
5167        let mut budget = 16 * 1024;
5168        let status = http_line(&mut reader, &mut budget).await?;
5169        let mut words = status.split_ascii_whitespace();
5170        let version = words.next();
5171        let code = words
5172            .next()
5173            .filter(|word| word.len() == 3)
5174            .and_then(|word| word.parse::<u16>().ok());
5175        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5176            || !code.is_some_and(|code| (100..600).contains(&code))
5177        {
5178            return Err(HealthProbeError::bad_answer(format!(
5179                "invalid HTTP status: {status}"
5180            )));
5181        }
5182        let code = code.expect("validated status code");
5183        response_status = status.clone();
5184        let mut length = None;
5185        let mut chunked = false;
5186        loop {
5187            let line = http_line(&mut reader, &mut budget).await?;
5188            if line.is_empty() {
5189                break;
5190            }
5191            if let Some((name, value)) = line.split_once(':') {
5192                if name.eq_ignore_ascii_case("content-length") {
5193                    length = Some(value.trim().parse::<u64>().map_err(|_| {
5194                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
5195                    })?);
5196                } else if name.eq_ignore_ascii_case("transfer-encoding") {
5197                    chunked = value.trim().eq_ignore_ascii_case("chunked");
5198                }
5199            }
5200        }
5201        if chunked {
5202            while body.len() < 200 {
5203                let line = http_line(&mut reader, &mut budget).await?;
5204                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5205                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5206                if size == 0 {
5207                    break;
5208                }
5209                let count = size.min((200 - body.len()) as u64) as usize;
5210                let start = body.len();
5211                (&mut reader)
5212                    .take(count as u64)
5213                    .read_to_end(&mut body)
5214                    .await
5215                    .map_err(|error| {
5216                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5217                    })?;
5218                if body.len() - start != count {
5219                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5220                }
5221                if size > count as u64 || body.len() == 200 {
5222                    break;
5223                }
5224                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5225                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5226                }
5227            }
5228        } else {
5229            reader
5230                .take(length.unwrap_or(200).min(200))
5231                .read_to_end(&mut body)
5232                .await
5233                .map_err(|error| {
5234                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5235                })?;
5236        }
5237        if (200..300).contains(&code) {
5238            Ok(HealthReport::ok())
5239        } else {
5240            Err(HealthProbeError::bad_answer(
5241                "HTTP health endpoint returned non-2xx",
5242            ))
5243        }
5244    };
5245    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5246        Err(HealthProbeError::no_answer(format!(
5247            "HTTP probe timed out after {deadline:?}"
5248        )))
5249    });
5250    if let Err(error) = &mut result {
5251        if !response_status.is_empty() {
5252            error.message = format!(
5253                "{}; {response_status}: {}",
5254                error.message,
5255                String::from_utf8_lossy(&body)
5256            );
5257        }
5258    }
5259    result
5260}
5261
5262async fn http_line(
5263    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5264    remaining: &mut usize,
5265) -> Result<String, HealthProbeError> {
5266    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5267    let mut line = Vec::new();
5268    (&mut *reader)
5269        .take(*remaining as u64)
5270        .read_until(b'\n', &mut line)
5271        .await
5272        .map_err(|error| {
5273            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5274        })?;
5275    *remaining -= line.len();
5276    if !line.ends_with(b"\r\n") {
5277        return Err(HealthProbeError::bad_answer(
5278            "HTTP headers are incomplete or exceed 16 KiB",
5279        ));
5280    }
5281    line.truncate(line.len() - 2);
5282    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5283}
5284
5285async fn probe_module_health(
5286    module_id: &str,
5287    runtime: &SupervisorRuntimeConfig,
5288    drain_deadline: Option<Instant>,
5289) -> Result<HealthReport, HealthProbeError> {
5290    let Some(forwarding) = runtime.forwarding.as_ref() else {
5291        return Err(HealthProbeError::misconfigured(
5292            "supervisor was not configured with a forwarding table",
5293        ));
5294    };
5295    let probe_started_at = Instant::now();
5296    let mut deadline = probe_started_at + runtime.health.deadline;
5297    if let Some(drain_deadline) = drain_deadline {
5298        deadline = deadline.min(drain_deadline);
5299    }
5300    let pending = if drain_deadline.is_some() {
5301        forwarding.begin_drain_health_probe_rpc_for(
5302            module_id,
5303            MODULE_CONTROL_OP_HEALTH_CHECK,
5304            probe_started_at,
5305            deadline,
5306        )
5307    } else {
5308        forwarding.begin_health_probe_rpc_for(
5309            module_id,
5310            MODULE_CONTROL_OP_HEALTH_CHECK,
5311            probe_started_at,
5312            deadline,
5313        )
5314    }
5315    .map_err(|err| {
5316        // The endpoint is not registered, so there is no live control lane to
5317        // ask. That is the module being absent, not slow.
5318        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5319    })?;
5320    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5321}
5322
5323/// [`probe_module_health`] for one endpoint rather than the id's active one.
5324///
5325/// A swap probes two processes that no by-id lookup reaches: its candidate
5326/// before cutover, and its superseded incumbent (for busy gauges) while the
5327/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5328/// bounds the by-id drain probe.
5329async fn probe_endpoint_health(
5330    endpoint: crate::ModuleEndpointId,
5331    runtime: &SupervisorRuntimeConfig,
5332    deadline_cap: Option<Instant>,
5333) -> Result<HealthReport, HealthProbeError> {
5334    let Some(forwarding) = runtime.forwarding.as_ref() else {
5335        return Err(HealthProbeError::misconfigured(
5336            "supervisor was not configured with a forwarding table",
5337        ));
5338    };
5339    let probe_started_at = Instant::now();
5340    let mut deadline = probe_started_at + runtime.health.deadline;
5341    if let Some(cap) = deadline_cap {
5342        deadline = deadline.min(cap);
5343    }
5344    let pending = forwarding
5345        .begin_endpoint_health_probe_rpc_for(
5346            endpoint,
5347            MODULE_CONTROL_OP_HEALTH_CHECK,
5348            probe_started_at,
5349            deadline,
5350        )
5351        .map_err(|err| {
5352            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5353        })?;
5354    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5355}
5356
5357/// Send a begun health probe and classify its answer.
5358async fn await_health_probe(
5359    forwarding: &ForwardingTable,
5360    pending: PendingModuleControlRpc,
5361    deadline: Instant,
5362    probe_budget: Duration,
5363) -> Result<HealthReport, HealthProbeError> {
5364    let PendingModuleControlRpc {
5365        endpoint,
5366        module_sink,
5367        negotiated_ver,
5368        corr,
5369        receiver,
5370    } = pending;
5371    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5372        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5373    })?;
5374    let frame = Frame::build_with_version(
5375        negotiated_ver,
5376        FrameType::Request,
5377        control_flags(),
5378        0,
5379        0,
5380        corr,
5381        body,
5382    )
5383    .map_err(|err| {
5384        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5385    })?;
5386
5387    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5388    // blocks waiting for capacity when the module's egress queue is full, and an
5389    // unbounded await here freezes the whole supervision actor (it stops polling
5390    // Child::wait and supervisor commands), making the module unrecoverable
5391    // in-band. On timeout the probe fails like any transport failure.
5392    match timeout_at(deadline, module_sink.send(frame)).await {
5393        Ok(Ok(())) => {}
5394        Ok(Err(err)) => {
5395            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5396            // A closed sink means the module's egress channel is gone -- the
5397            // receiving half is dropped when its connection tears down. Proof.
5398            return Err(HealthProbeError::lane_dead(format!(
5399                "failed to send health.check: {err}"
5400            )));
5401        }
5402        Err(_elapsed) => {
5403            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5404            // A full egress queue means the module is not draining its socket, which
5405            // is consistent with a wedged module AND with one whose reader is merely
5406            // starved. Silence, not proof.
5407            return Err(HealthProbeError::no_answer(
5408                "health.check send timed out before enqueue (module egress full)",
5409            ));
5410        }
5411    }
5412
5413    match timeout_at(deadline, receiver).await {
5414        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5415        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5416        // and those prove it is alive even though the probe failed.
5417        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5418            response.health_report().ok_or_else(|| {
5419                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5420            })
5421        }
5422        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5423            format!("health.check rejected: {}", body.message),
5424        )),
5425        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5426            Err(HealthProbeError::lane_dead(message))
5427        }
5428        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5429            Err(HealthProbeError::bad_answer(message))
5430        }
5431        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5432            Err(HealthProbeError::bad_answer(format!(
5433                "expected module-control op '{expected}', got '{actual}'"
5434            )))
5435        }
5436        // A reply that crosses the deadline before this waiter observes it is
5437        // still proof of life. The forwarding path records its end-to-end latency
5438        // before delivering this classification.
5439        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5440            "module answered health.check after its daemon deadline",
5441        )),
5442        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5443            "health.check waiter was canceled before the module responded",
5444        )),
5445        Err(_) => {
5446            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5447            Err(HealthProbeError::no_answer(format!(
5448                "module did not answer health.check within {probe_budget:?}"
5449            )))
5450        }
5451    }
5452}
5453
5454#[allow(clippy::too_many_arguments)]
5455async fn handle_health_report(
5456    spec: &ModuleSpec,
5457    runtime: &SupervisorRuntimeConfig,
5458    registry: &Registry,
5459    process_liveness: &SupervisorProcessLiveness,
5460    snapshot: &SharedSnapshot,
5461    child: &mut Option<SupervisedChild>,
5462    report: HealthReport,
5463    now_ms: u64,
5464) {
5465    let status = supervisor_health_status(report.status);
5466    let detail = report.detail.clone();
5467    let metrics = truncate_health_metrics(report.metrics);
5468    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5469        state.health.status = status;
5470        state.health.last_probe_ms = Some(now_ms);
5471        state.health.detail = detail.clone();
5472        state.health.metrics = metrics.clone();
5473        state.health.consecutive_failures = 0;
5474        state.failed_retry.observe_health(
5475            report.status == HealthStatus::Ok,
5476            runtime.restart_policy.window,
5477        );
5478    });
5479
5480    let action = match report.status {
5481        HealthStatus::Ok => return,
5482        HealthStatus::Degraded => runtime.health.on_degraded,
5483        HealthStatus::Failing => runtime.health.on_failing,
5484    };
5485    apply_l3_health_action(
5486        spec,
5487        runtime,
5488        registry,
5489        process_liveness,
5490        snapshot,
5491        child,
5492        status,
5493        detail.as_deref(),
5494        action,
5495        now_ms,
5496    )
5497    .await;
5498}
5499
5500#[allow(clippy::too_many_arguments)]
5501async fn handle_health_probe_failure(
5502    spec: &ModuleSpec,
5503    runtime: &SupervisorRuntimeConfig,
5504    registry: &Registry,
5505    process_liveness: &SupervisorProcessLiveness,
5506    snapshot: &SharedSnapshot,
5507    child: &mut Option<SupervisedChild>,
5508    err: HealthProbeError,
5509    now_ms: u64,
5510) {
5511    let threshold = runtime.health.failure_threshold.max(1);
5512    let mut failures = 0;
5513    // Carry the evidence class into the operator-visible detail. Without it,
5514    // "module did not answer within 5s" and "the control lane is gone" are two
5515    // prose strings in the same field, and the reader has to know the codebase to
5516    // tell which one is proof of anything.
5517    let detail = format!("[{}] {err}", err.label());
5518    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5519        state
5520            .failed_retry
5521            .observe_health(false, runtime.restart_policy.window);
5522        // A failed wire probe invalidates the last report, even before the
5523        // restart threshold. HTTP probes already mark failures as Failing.
5524        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5525            state.health.status = SupervisorHealthStatus::Unknown;
5526        }
5527        state.health.last_probe_ms = Some(now_ms);
5528        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5529        state.health.detail = Some(detail.clone());
5530        state.health.metrics = None;
5531        failures = state.health.consecutive_failures;
5532    });
5533
5534    if failures < threshold {
5535        warn!(
5536            module_id = %spec.module_id,
5537            consecutive_failures = failures,
5538            threshold,
5539            evidence = err.label(),
5540            detail = %detail,
5541            "health.check probe failed"
5542        );
5543        return;
5544    }
5545
5546    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5547        state.state = ModuleState::Unresponsive;
5548        state.health.status = SupervisorHealthStatus::Unresponsive;
5549    });
5550    // The evidence class is logged at the kill site because this is the line an
5551    // operator reads after an unexplained restart. A streak of `no-answer` under
5552    // machine load is the known false-positive shape; a `lane-dead` is not.
5553    if runtime.health.critical {
5554        error!(
5555            module_id = %spec.module_id,
5556            status = "unresponsive",
5557            evidence = err.label(),
5558            detail = %detail,
5559            "critical module health alert"
5560        );
5561    } else {
5562        warn!(
5563            module_id = %spec.module_id,
5564            status = "unresponsive",
5565            evidence = err.label(),
5566            detail = %detail,
5567            "module health threshold breached"
5568        );
5569    }
5570    if let Err(err) = health_restart_child(
5571        spec,
5572        runtime,
5573        registry,
5574        process_liveness,
5575        snapshot,
5576        child,
5577        SupervisorHealthStatus::Unresponsive,
5578        Some(&detail),
5579        now_ms,
5580    )
5581    .await
5582    {
5583        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5584    }
5585}
5586
5587#[allow(clippy::too_many_arguments)]
5588async fn apply_l3_health_action(
5589    spec: &ModuleSpec,
5590    runtime: &SupervisorRuntimeConfig,
5591    registry: &Registry,
5592    process_liveness: &SupervisorProcessLiveness,
5593    snapshot: &SharedSnapshot,
5594    child: &mut Option<SupervisedChild>,
5595    status: SupervisorHealthStatus,
5596    detail: Option<&str>,
5597    action: HealthAction,
5598    now_ms: u64,
5599) {
5600    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5601    match action {
5602        HealthAction::Report => {
5603            info!(
5604                module_id = %spec.module_id,
5605                status = ?status,
5606                detail,
5607                "module reported non-ok health"
5608            );
5609        }
5610        HealthAction::Alert => {
5611            error!(
5612                module_id = %spec.module_id,
5613                status = ?status,
5614                detail,
5615                "module health alert"
5616            );
5617        }
5618        HealthAction::Restart => {
5619            if let Err(err) = health_restart_child(
5620                spec,
5621                runtime,
5622                registry,
5623                process_liveness,
5624                snapshot,
5625                child,
5626                status,
5627                detail,
5628                now_ms,
5629            )
5630            .await
5631            {
5632                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5633            }
5634        }
5635    }
5636}
5637
5638#[allow(clippy::too_many_arguments)]
5639async fn health_restart_child(
5640    spec: &ModuleSpec,
5641    runtime: &SupervisorRuntimeConfig,
5642    registry: &Registry,
5643    process_liveness: &SupervisorProcessLiveness,
5644    snapshot: &SharedSnapshot,
5645    child: &mut Option<SupervisedChild>,
5646    status: SupervisorHealthStatus,
5647    detail: Option<&str>,
5648    now_ms: u64,
5649) -> Result<(), SuperviseError> {
5650    let (enabled, schedule) = {
5651        let mut state = lock_snapshot(snapshot)?;
5652        let enabled = state.enabled;
5653        let schedule = if enabled {
5654            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5655        } else {
5656            None
5657        };
5658        (enabled, schedule)
5659    };
5660
5661    if !enabled {
5662        return Err(SuperviseError::Disabled {
5663            module_id: spec.module_id.clone(),
5664        });
5665    }
5666
5667    if schedule.is_none() {
5668        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5669        error!(
5670            module_id = %spec.module_id,
5671            status = ?status,
5672            detail,
5673            max_restarts = runtime.restart_policy.max_restarts,
5674            window_secs = runtime.restart_policy.window.as_secs(),
5675            reason = %runtime.restart_policy.budget_exhausted_detail(),
5676            "health restart budget exhausted; marking module failed"
5677        );
5678        let stop_notice = begin_forwarding_drain_if_configured(
5679            spec,
5680            runtime,
5681            registry,
5682            snapshot,
5683            Some(true),
5684            RouteCloseReason::Disable,
5685        )
5686        .await?;
5687        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5688            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5689        })?;
5690        let result = drain_optional_child(
5691            &spec.module_id,
5692            spec.protocol,
5693            stop_notice,
5694            registry,
5695            runtime.forwarding.as_deref(),
5696            snapshot,
5697            &runtime.terminal_ring,
5698            &runtime.spawn_events,
5699            child,
5700            runtime.drain_timeout,
5701            ModuleState::Failed,
5702            Some(true),
5703            &runtime.failure_context(true),
5704        )
5705        .await;
5706        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5707            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5708                state.failed_retry.schedule(&spec.module_id);
5709            }
5710        })?;
5711        result?;
5712        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5713        return Ok(());
5714    }
5715
5716    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5717    let mut restart_count = 0;
5718    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5719        restart_count = state.crash_restarts.len();
5720        state.state = ModuleState::Unresponsive;
5721        state.health.status = status;
5722        state.health.last_action = Some(HealthAction::Restart.to_string());
5723        state.health.last_action_ms = Some(now_ms);
5724    })?;
5725    warn!(
5726        module_id = %spec.module_id,
5727        status = ?status,
5728        detail,
5729        restart_count,
5730        restart_in_window = schedule.restart_in_window,
5731        delay_ms = schedule.delay.as_millis() as u64,
5732        "health-triggered module restart"
5733    );
5734
5735    let stop_notice = begin_forwarding_drain_if_configured(
5736        spec,
5737        runtime,
5738        registry,
5739        snapshot,
5740        Some(true),
5741        RouteCloseReason::Restart,
5742    )
5743    .await?;
5744    drain_optional_child(
5745        &spec.module_id,
5746        spec.protocol,
5747        stop_notice,
5748        registry,
5749        runtime.forwarding.as_deref(),
5750        snapshot,
5751        &runtime.terminal_ring,
5752        &runtime.spawn_events,
5753        child,
5754        runtime.drain_timeout,
5755        ModuleState::Restarting,
5756        Some(true),
5757        &runtime.failure_context(true),
5758    )
5759    .await?;
5760    schedule_respawn(
5761        runtime,
5762        snapshot,
5763        &spec.module_id,
5764        schedule.delay,
5765        RespawnKind::Spawn,
5766        true,
5767    )
5768}
5769
5770fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5771    if let Some(reply) = runtime
5772        .deferred_reload_reply
5773        .lock()
5774        .unwrap_or_else(|p| p.into_inner())
5775        .take()
5776    {
5777        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5778            module_id: module_id.to_string(),
5779            reason: reason.to_string(),
5780        }));
5781    }
5782}
5783
5784fn schedule_respawn(
5785    runtime: &SupervisorRuntimeConfig,
5786    snapshot: &SharedSnapshot,
5787    module_id: &str,
5788    delay: Duration,
5789    kind: RespawnKind,
5790    daemon_initiated: bool,
5791) -> Result<(), SuperviseError> {
5792    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5793    update_snapshot(snapshot, Some(module_id), |state| {
5794        state.respawn_pending = true
5795    })?;
5796    *runtime
5797        .scheduled_respawn
5798        .lock()
5799        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5800        deadline: Instant::now() + delay,
5801        kind,
5802        daemon_initiated,
5803    });
5804    Ok(())
5805}
5806
5807fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5808    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5809        state.health.last_action = Some(action);
5810        state.health.last_action_ms = Some(now_ms);
5811    });
5812}
5813
5814fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5815    match status {
5816        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5817        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5818        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5819    }
5820}
5821
5822/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5823/// returned to every `supervisor.list` and `supervisor.health` caller.
5824///
5825/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5826/// path: that request exists to return a module's complete metrics object, and
5827/// `ck health <module-id>` documents it as the way to see what the cached view
5828/// truncates. The asymmetry is the feature.
5829///
5830/// So a new caller must decide which side it is on rather than assume the cap is
5831/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5832/// the truncation that path exists to avoid.
5833fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5834    let metrics = metrics?;
5835    match serde_json::to_vec(&metrics) {
5836        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5837            "truncated": true,
5838            "original_bytes": encoded.len(),
5839        })),
5840        Ok(_) | Err(_) => Some(metrics),
5841    }
5842}
5843
5844/// Spread health probes so a fleet-wide restart does not converge them.
5845///
5846/// The delay is derived from the module id and probe index rather than a random
5847/// source, so it is deterministic per module: a module keeps its own offset
5848/// across daemon restarts instead of re-rolling into a collision.
5849fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5850    if cadence.is_zero() {
5851        return Duration::ZERO;
5852    }
5853    let cadence_ms = cadence.as_millis() as u64;
5854    // This early return is REDUNDANT, deliberately, and a mutation run will show
5855    // it surviving removal. Recording why here so the next person to notice does
5856    // not have to re-derive it:
5857    //
5858    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5859    //   a zero cadence and builds the Duration from whole milliseconds, so a
5860    //   sub-millisecond cadence cannot come from config.
5861    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5862    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5863    //   -- exactly what this returns.
5864    //
5865    // Kept as a guard against a future widening of the config parser (accepting
5866    // microseconds, say), which would make the sub-millisecond case reachable.
5867    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5868    // divides by zero. Remove this and nothing changes.
5869    if cadence_ms == 0 {
5870        return cadence;
5871    }
5872    // Note that this never returns less than one cadence, including for the FIRST
5873    // probe. So a freshly registered module reports health `unknown` for a full
5874    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5875    // ready to answer.
5876    //
5877    // That is a property of the supervisor's schedule, not of any module: an
5878    // operator watching a restart sees `unknown` and cannot tell it from a module
5879    // that is slow to warm. Measured on two unrelated modules, both flipping to
5880    // `ok` between 22s and 32s after restart.
5881    //
5882    // Left as-is because spreading the first probe is what keeps a fleet-wide
5883    // restart from firing fourteen simultaneous probes into a cold machine. The
5884    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5885    // that thundering herd for a faster first reading.
5886    let jitter_span = (cadence_ms / 10).max(1);
5887    let hash = module_id.as_bytes().iter().fold(
5888        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5889        |acc, byte| {
5890            acc.wrapping_mul(1099511628211)
5891                .wrapping_add(u64::from(*byte))
5892        },
5893    );
5894    cadence + Duration::from_millis(hash % jitter_span)
5895}
5896
5897#[cfg(test)]
5898mod tests {
5899    use super::*;
5900
5901    #[test]
5902    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5903        let handle = SupervisorHandle::new();
5904        let module_id = "readded-tombstone";
5905        handle.record_rescan_removal(module_id);
5906        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5907
5908        handle.apply_identity_configuration(&ModuleSpec {
5909            module_id: module_id.to_string(),
5910            program: PathBuf::from("/test/module"),
5911            args: Vec::new(),
5912            env: Vec::new(),
5913            reserved: false,
5914            reserved_prefixes: Vec::new(),
5915            protocol: ModuleProtocol::Subc,
5916            overlap: Default::default(),
5917            carry_containment: false,
5918        });
5919
5920        assert!(
5921            handle.removal_tombstone_age_ms(module_id).is_none(),
5922            "a re-added module must not retain a stale removal tombstone"
5923        );
5924    }
5925
5926    /// What one module's owner looked like from the control plane at the
5927    /// instant after its first process was spawned.
5928    #[derive(Debug, PartialEq, Eq)]
5929    struct OwnerInSpawnWindow {
5930        module_id: String,
5931        configured: bool,
5932        on_roster: bool,
5933        admission_refusal: Option<&'static str>,
5934    }
5935
5936    /// A supervised module's process can connect, register, sync its scopes
5937    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5938    /// the module on the roster until afterward; configured supervision does
5939    /// so before spawning. In both cases the owner must
5940    /// already read as configured, so a scoped `route.open` against it is
5941    /// refused as retryable `scope_not_synced` and not as terminal
5942    /// `scope_not_live` ("will never sync").
5943    ///
5944    /// The hook runs just after each initial spawn attempt, before returning to
5945    /// the caller, so no race with a real child's registration is needed.
5946    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5947    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5948        use crate::scopes::ScopeTable;
5949        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5950
5951        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5952        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5953            module_id: module_id.to_string(),
5954            program,
5955            args: Vec::new(),
5956            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5957                .into_iter()
5958                .map(|key| (key.to_string(), dir.path().display().to_string()))
5959                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5960                .collect(),
5961            reserved: false,
5962            reserved_prefixes: Vec::new(),
5963            protocol: ModuleProtocol::Subc,
5964            overlap: Default::default(),
5965            carry_containment: false,
5966        };
5967        let live = super::terminal_history_tests::fake_aft_stub_path();
5968        let missing = dir.path().join("definitely-missing-module");
5969
5970        let handle = SupervisorHandle::new();
5971        let mut supervisor =
5972            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5973                .with_handle(handle.clone());
5974        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5975        let hook_handle = handle.clone();
5976        let hook_observed = Arc::clone(&observed);
5977        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5978            // Exactly what the control plane computes for a scoped route.open
5979            // naming this module as the owner of a scope it has not synced.
5980            let configured = hook_handle.is_configured(module_id);
5981            let selector = ScopeSelector::new(
5982                Principal::Reserved {
5983                    module_id: module_id.to_string(),
5984                },
5985                "s".to_string(),
5986                Some(1),
5987            );
5988            let carrier = Principal::Reserved {
5989                module_id: "carrier".to_string(),
5990            };
5991            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5992                .admit(&carrier, module_id, &selector, configured)
5993            {
5994                Ok(_) => None,
5995                Err(refusal) => Some(refusal.code),
5996            };
5997            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5998                module_id: module_id.to_string(),
5999                configured,
6000                on_roster: hook_handle.get(module_id).is_some(),
6001                admission_refusal,
6002            });
6003        })));
6004
6005        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
6006        let configured = supervisor
6007            .supervise_configured(stub("configured", live.clone()), true)
6008            .unwrap();
6009        let with_health = supervisor
6010            .supervise_configured_with_health(
6011                stub("with-health", live.clone()),
6012                true,
6013                HealthConfig::default(),
6014                None,
6015                RestartPolicy::default(),
6016            )
6017            .unwrap();
6018        // The failed-spawn path still puts the module on the roster (as
6019        // failed), so it is configured throughout.
6020        let failed = supervisor
6021            .supervise_configured_with_health(
6022                stub("failed-spawn", missing.clone()),
6023                true,
6024                HealthConfig::default(),
6025                None,
6026                RestartPolicy::default(),
6027            )
6028            .unwrap();
6029        // A failed plain `spawn` puts nothing on the roster, so its mark is
6030        // taken back once the spawn has failed.
6031        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6032
6033        let expected = [
6034            "plain",
6035            "configured",
6036            "with-health",
6037            "failed-spawn",
6038            "spawn-error",
6039        ]
6040        .into_iter()
6041        .map(|module_id| OwnerInSpawnWindow {
6042            module_id: module_id.to_string(),
6043            configured: true,
6044            on_roster: !matches!(module_id, "plain" | "spawn-error"),
6045            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6046        })
6047        .collect::<Vec<_>>();
6048        assert_eq!(*observed.lock().unwrap(), expected);
6049
6050        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6051            assert!(
6052                handle.get(module_id).is_some(),
6053                "{module_id} is on the roster"
6054            );
6055            assert!(
6056                handle.is_configured(module_id),
6057                "{module_id} stays configured"
6058            );
6059        }
6060        assert!(handle.get("spawn-error").is_none());
6061        assert!(
6062            !handle.is_configured("spawn-error"),
6063            "a plain spawn that failed must not leave its module marked configured"
6064        );
6065
6066        // Leaving the roster clears the mark with it.
6067        handle.retire("failed-spawn");
6068        assert!(!handle.is_configured("failed-spawn"));
6069
6070        for module in [plain, configured, with_health] {
6071            module.stop().await.unwrap();
6072        }
6073        drop(failed);
6074    }
6075
6076    /// A configured module reads as starting until the supervisor has admitted
6077    /// its first process (exec confirmed, identity durably recorded), even
6078    /// though that process is already running by then and can register over
6079    /// HELLO, which is what lists it in catalog.list. So a client that needs
6080    /// the supervisor state must wait for that state; seeing the module in
6081    /// the catalog does not mean supervisor.list already says running.
6082    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6083    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6084        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6085        let spec = ModuleSpec {
6086            module_id: "spawned-not-admitted".to_string(),
6087            program: super::terminal_history_tests::fake_aft_stub_path(),
6088            args: Vec::new(),
6089            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6090                .into_iter()
6091                .map(|key| (key.to_string(), dir.path().display().to_string()))
6092                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6093                .collect(),
6094            reserved: false,
6095            reserved_prefixes: Vec::new(),
6096            protocol: ModuleProtocol::Subc,
6097            overlap: Default::default(),
6098            carry_containment: false,
6099        };
6100        let handle = SupervisorHandle::new();
6101        let mut supervisor =
6102            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6103                .with_handle(handle.clone());
6104        let observed = Arc::new(Mutex::new(None));
6105        let hook_handle = handle.clone();
6106        let hook_observed = Arc::clone(&observed);
6107        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6108            let state = hook_handle
6109                .get(module_id)
6110                .expect("a configured module is on the roster before it is spawned")
6111                .state()
6112                .unwrap();
6113            *hook_observed.lock().unwrap() = Some(state);
6114        })));
6115
6116        let module = supervisor.supervise_configured(spec, true).unwrap();
6117        assert_eq!(
6118            *observed.lock().unwrap(),
6119            Some(ModuleState::Starting),
6120            "between the spawn and its admission the supervisor still reports starting"
6121        );
6122        assert_eq!(module.state().unwrap(), ModuleState::Running);
6123        module.stop().await.unwrap();
6124    }
6125
6126    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6127        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6128        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6129            snapshot.process_alive = true;
6130            snapshot.pid = Some(41);
6131            snapshot.spawned_at_ms = Some(42);
6132            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6133            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6134            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6135                device: 43,
6136                inode: 44,
6137                #[cfg(windows)]
6138                windows_image: None,
6139            });
6140        })
6141        .unwrap();
6142        snapshot
6143    }
6144
6145    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6146        let snapshot = lock_snapshot(snapshot).unwrap();
6147        assert!(!snapshot.process_alive);
6148        assert_eq!(snapshot.pid, None);
6149        assert_eq!(snapshot.spawned_at_ms, None);
6150        assert_eq!(snapshot.spawned_from, None);
6151        assert_eq!(snapshot.launch_spec_digest, None);
6152        assert_eq!(snapshot.spawned_file_identity, None);
6153    }
6154
6155    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6156    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6157        let supervisor =
6158            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6159        let mut runtime = supervisor.runtime_config();
6160        runtime.test_seed_stale_facts_before_enable_spawn = true;
6161        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6162        let mut child = None;
6163        let spec = ModuleSpec {
6164            module_id: "failed-enable-clears-facts".to_string(),
6165            program: PathBuf::from("/definitely/missing/failed-enable-module"),
6166            args: Vec::new(),
6167            env: Vec::new(),
6168            reserved: false,
6169            reserved_prefixes: Vec::new(),
6170            protocol: ModuleProtocol::Subc,
6171            overlap: Default::default(),
6172            carry_containment: false,
6173        };
6174
6175        let result = set_child_enabled(
6176            &spec,
6177            &runtime,
6178            &supervisor.registry,
6179            &supervisor.process_liveness,
6180            &snapshot,
6181            &mut child,
6182            true,
6183            &runtime.failure_context(false),
6184        )
6185        .await;
6186
6187        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6188        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6189        assert_snapshot_process_facts_cleared(&snapshot);
6190    }
6191
6192    #[tokio::test]
6193    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6194        let supervisor =
6195            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6196        let runtime = supervisor.runtime_config();
6197        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6198            ModuleState::Restarting,
6199            true,
6200        )));
6201        let spec = ModuleSpec {
6202            module_id: "start-stranded-restarting".to_string(),
6203            program: super::terminal_history_tests::fake_aft_stub_path(),
6204            args: Vec::new(),
6205            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6206            reserved: false,
6207            reserved_prefixes: Vec::new(),
6208            protocol: ModuleProtocol::None,
6209            overlap: Default::default(),
6210            carry_containment: false,
6211        };
6212        let mut child = None;
6213        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6214        assert!(!super::set_child_enabled(
6215            &spec,
6216            &runtime,
6217            &Registry::default(),
6218            &supervisor.process_liveness,
6219            &snapshot,
6220            &mut child,
6221            true,
6222            &runtime.failure_context(false),
6223        )
6224        .await
6225        .unwrap());
6226        assert!(child.is_none());
6227        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6228        assert!(super::set_child_enabled(
6229            &spec,
6230            &runtime,
6231            &Registry::default(),
6232            &supervisor.process_liveness,
6233            &snapshot,
6234            &mut child,
6235            true,
6236            &runtime.failure_context(false),
6237        )
6238        .await
6239        .unwrap());
6240        assert_eq!(
6241            lock_snapshot(&snapshot).unwrap().state,
6242            ModuleState::Running
6243        );
6244        let mut child = child.unwrap();
6245        child.start_kill().unwrap();
6246        child.wait().await.unwrap();
6247    }
6248
6249    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6250    async fn failed_reload_spawn_clears_current_process_facts() {
6251        let supervisor =
6252            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6253        let mut runtime = supervisor.runtime_config();
6254        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6255        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6256        let mut child = None;
6257        let spec = ModuleSpec {
6258            module_id: "failed-reload-clears-facts".to_string(),
6259            program: PathBuf::from("/unused/failed-reload-module"),
6260            args: Vec::new(),
6261            env: Vec::new(),
6262            reserved: false,
6263            reserved_prefixes: Vec::new(),
6264            protocol: ModuleProtocol::Subc,
6265            overlap: Default::default(),
6266            carry_containment: false,
6267        };
6268
6269        let result = handle_reload_spawn_failure(
6270            &spec,
6271            &runtime,
6272            &supervisor.process_liveness,
6273            &snapshot,
6274            &mut child,
6275            "forced reload spawn failure".to_string(),
6276        )
6277        .await;
6278
6279        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6280        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6281        assert_snapshot_process_facts_cleared(&snapshot);
6282    }
6283
6284    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6285    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6286        let supervisor =
6287            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6288        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6289        let module = supervisor.supervised_module(
6290            ModuleSpec {
6291                module_id: "drop-clears-facts".to_string(),
6292                program: PathBuf::from("/unused/drop-module"),
6293                args: Vec::new(),
6294                env: Vec::new(),
6295                reserved: false,
6296                reserved_prefixes: Vec::new(),
6297                protocol: ModuleProtocol::Subc,
6298                overlap: Default::default(),
6299                carry_containment: false,
6300            },
6301            supervisor.runtime_config(),
6302            Arc::clone(&snapshot),
6303            None,
6304        );
6305        assert!(!module
6306            .inner
6307            .monitor
6308            .lock()
6309            .unwrap()
6310            .as_ref()
6311            .unwrap()
6312            .is_finished());
6313
6314        drop(module);
6315
6316        assert_eq!(
6317            lock_snapshot(&snapshot).unwrap().state,
6318            ModuleState::Stopped
6319        );
6320        assert_snapshot_process_facts_cleared(&snapshot);
6321    }
6322
6323    #[cfg(unix)]
6324    #[tokio::test]
6325    async fn rescan_preserves_running_protocol_until_respawn() {
6326        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6327        let initial = ModuleSpec {
6328            module_id: "rescan-protocol".into(),
6329            program: PathBuf::from("/bin/sleep"),
6330            args: vec!["60".into()],
6331            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6332                .into_iter()
6333                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6334                .collect(),
6335            reserved: false,
6336            reserved_prefixes: vec![],
6337            protocol: ModuleProtocol::None,
6338            overlap: Default::default(),
6339            carry_containment: false,
6340        };
6341        let supervisor =
6342            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6343        let module = supervisor.spawn(initial.clone()).unwrap();
6344        assert!(module.status().unwrap().live);
6345        let mut next = initial;
6346        next.protocol = ModuleProtocol::Subc;
6347        module
6348            .update_configuration(next.clone(), HealthConfig::default(), None)
6349            .await
6350            .unwrap();
6351        assert!(
6352            module.status().unwrap().live,
6353            "rescan must not require HELLO from the old non-wire process"
6354        );
6355        let runtime = supervisor.runtime_config();
6356        let action = on_child_exit(
6357            &next,
6358            RestartPolicy::default(),
6359            &supervisor.registry,
6360            &module.inner.snapshot,
6361            &runtime.terminal_ring,
6362            &runtime.spawn_events,
6363            &runtime.child_roster,
6364            ExitReport {
6365                kind: ExitKind::Clean,
6366                code: Some(0),
6367                signal: None,
6368                at_ms: unix_ms_now(),
6369            },
6370            true,
6371        )
6372        .await;
6373        assert!(
6374            matches!(action, NextAction::Restart { .. }),
6375            "the old non-wire process's clean exit must restart"
6376        );
6377        module.drain().await.unwrap();
6378    }
6379
6380    #[cfg(unix)]
6381    #[tokio::test]
6382    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6383        use std::os::unix::fs::PermissionsExt;
6384        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6385        let script = dir.join("module.sh");
6386        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6387        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6388        let record_path = dir.join("live-children.json");
6389        let supervisor =
6390            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6391                .with_live_children_record(&record_path);
6392        for (program, args) in [
6393            (PathBuf::from("sleep"), vec!["60".into()]),
6394            (script, vec![]),
6395        ] {
6396            let spec = ModuleSpec {
6397                module_id: "image-identity".into(),
6398                program: program.clone(),
6399                args,
6400                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6401                    .into_iter()
6402                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6403                    .collect(),
6404                reserved: false,
6405                reserved_prefixes: vec![],
6406                protocol: ModuleProtocol::None,
6407                overlap: Default::default(),
6408                carry_containment: false,
6409            };
6410            let module = supervisor.spawn(spec).unwrap();
6411            #[cfg(target_os = "macos")]
6412            {
6413                // SETEXEC confirmation is asynchronous; the orphan record must
6414                // identify the final image, never the intermediate trampoline.
6415                let deadline = Instant::now() + Duration::from_secs(5);
6416                while crate::live_children::read_record(&record_path)
6417                    .unwrap()
6418                    .iter()
6419                    .all(|entry| entry.executable.is_none())
6420                {
6421                    assert!(Instant::now() < deadline, "module image was not confirmed");
6422                    tokio::time::sleep(Duration::from_millis(5)).await;
6423                }
6424            }
6425            let entry = crate::live_children::read_record(&record_path)
6426                .unwrap()
6427                .pop()
6428                .unwrap();
6429            let observed = subc_os::Process::open(entry.pid)
6430                .unwrap()
6431                .unwrap()
6432                .observe()
6433                .unwrap();
6434            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6435            module.drain().await.unwrap();
6436            assert_eq!(
6437                verdict,
6438                crate::live_children::IdentityVerdict::Matches,
6439                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6440            );
6441        }
6442    }
6443
6444    #[cfg(unix)]
6445    fn http_fixture(
6446        dir: &std::path::Path,
6447        url: &str,
6448        threshold: u32,
6449    ) -> crate::daemon_config::ConfiguredModule {
6450        let path = dir.join("subc.jsonc");
6451        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6452            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6453            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6454            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6455            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6456        }}}).to_string()).unwrap();
6457        crate::daemon_config::load(&path)
6458            .unwrap()
6459            .unwrap()
6460            .modules
6461            .pop()
6462            .unwrap()
6463    }
6464
6465    #[cfg(unix)]
6466    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6467        timeout(Duration::from_secs(5), async {
6468            loop {
6469                if module.status().unwrap().health.status == status {
6470                    break;
6471                }
6472                sleep(Duration::from_millis(5)).await;
6473            }
6474        })
6475        .await
6476        .unwrap_or_else(|_| {
6477            panic!(
6478                "expected {status:?}, got {:?}",
6479                module.status().unwrap().health
6480            )
6481        });
6482    }
6483
6484    #[cfg(unix)]
6485    #[tokio::test]
6486    async fn http_health_status_flips_ok_failing_ok() {
6487        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6488        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6489        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6490        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6491        let serving_status = status.clone();
6492        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6493        let server = tokio::spawn(async move {
6494            loop {
6495                let (mut stream, _) = listener.accept().await.unwrap();
6496                let mut request = [0u8; 2048];
6497                let count = stream.read(&mut request).await.unwrap();
6498                assert!(count > 0, "a probe must send an HTTP request");
6499                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6500                let body = if code == 200 {
6501                    "ready"
6502                } else {
6503                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6504                };
6505                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6506                let _ = stream.write_all(response.as_bytes()).await;
6507            }
6508        });
6509        let configured = http_fixture(&dir, &url, 1000);
6510        let module =
6511            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6512                .supervise_configured_with_health(
6513                    configured.module_spec(),
6514                    true,
6515                    configured.health,
6516                    configured.drain_timeout_ms,
6517                    configured.restart,
6518                )
6519                .unwrap();
6520        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6521        status.store(503, std::sync::atomic::Ordering::SeqCst);
6522        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6523        assert!(module
6524            .status()
6525            .unwrap()
6526            .health
6527            .detail
6528            .unwrap()
6529            .contains("scratch failure"));
6530        status.store(200, std::sync::atomic::Ordering::SeqCst);
6531        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6532        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6533        let before = module.status().unwrap();
6534        let (spec, mut health) = module.configuration().unwrap();
6535        health.http = None;
6536        module
6537            .update_configuration(spec.clone(), health.clone(), Some(10))
6538            .await
6539            .unwrap();
6540        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6541        health.http = Some(url);
6542        module
6543            .update_configuration(spec, health, Some(10))
6544            .await
6545            .unwrap();
6546        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6547        assert_eq!(
6548            module.status().unwrap().pid,
6549            before.pid,
6550            "changing a probe must apply live, not restart its process"
6551        );
6552        let (spec, mut health) = module.configuration().unwrap();
6553        health.failure_threshold = 2;
6554        module
6555            .update_configuration(spec, health, Some(10))
6556            .await
6557            .unwrap();
6558        status.store(503, std::sync::atomic::Ordering::SeqCst);
6559        timeout(Duration::from_secs(5), async {
6560            while module.status().unwrap().spawn_generation == before.spawn_generation {
6561                sleep(Duration::from_millis(5)).await;
6562            }
6563        })
6564        .await
6565        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6566        module.drain().await.unwrap();
6567        server.abort();
6568    }
6569
6570    #[cfg(unix)]
6571    #[tokio::test]
6572    async fn http_health_replacement_recovers_after_one_restart() {
6573        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6574        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6575        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6576        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6577        let mut configured = http_fixture(&dir, &url, 2);
6578        configured.restart.max_restarts = 2;
6579        let module =
6580            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6581                .supervise_configured_with_health(
6582                    configured.module_spec(),
6583                    true,
6584                    configured.health,
6585                    configured.drain_timeout_ms,
6586                    configured.restart,
6587                )
6588                .unwrap();
6589        let first_generation = module.status().unwrap().spawn_generation;
6590        let observed = module.clone();
6591        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6592        let server = tokio::spawn(async move {
6593            let mut generation = first_generation;
6594            let mut probes = 0;
6595            loop {
6596                let (mut stream, _) = listener.accept().await.unwrap();
6597                let mut request = [0; 2048];
6598                assert!(stream.read(&mut request).await.unwrap() > 0);
6599                let current = observed.status().unwrap().spawn_generation;
6600                if current != generation {
6601                    generation = current;
6602                    probes = 0;
6603                }
6604                probes += 1;
6605                // The original process never recovers. Each replacement has one
6606                // transient miss, then answers successfully for its remaining life.
6607                let healthy = generation != first_generation && probes > 1;
6608                let code = if healthy { 200 } else { 503 };
6609                let response = format!(
6610                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6611                );
6612                let _ = stream.write_all(response.as_bytes()).await;
6613                if healthy {
6614                    healthy_tx.send(()).await.unwrap();
6615                }
6616            }
6617        });
6618        timeout(Duration::from_secs(5), async {
6619            let mut healthy_probes = 0;
6620            loop {
6621                if module.state().unwrap() == ModuleState::Failed {
6622                    break;
6623                }
6624                while healthy_rx.try_recv().is_ok() {
6625                    healthy_probes += 1;
6626                }
6627                if healthy_probes >= 3
6628                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6629                {
6630                    break;
6631                }
6632                sleep(Duration::from_millis(5)).await;
6633            }
6634        })
6635        .await
6636        .expect("the replacement must settle healthy without exhausting its restart budget");
6637        let status = module.status().unwrap();
6638        module.drain().await.unwrap();
6639        server.abort();
6640        assert_eq!(status.state, ModuleState::Running);
6641        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6642        assert_eq!(status.health.consecutive_failures, 0);
6643        assert_eq!(status.lifetime_restarts, 1);
6644        assert_eq!(status.restart_count, 1);
6645    }
6646
6647    #[cfg(unix)]
6648    #[tokio::test]
6649    async fn http_health_sustained_failure_exhausts_restart_budget() {
6650        use std::collections::BTreeMap;
6651        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6652        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6653        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6654        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6655        let mut configured = http_fixture(&dir, &url, 2);
6656        configured.restart.max_restarts = 2;
6657        let module =
6658            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6659                .supervise_configured_with_health(
6660                    configured.module_spec(),
6661                    true,
6662                    configured.health,
6663                    configured.drain_timeout_ms,
6664                    configured.restart,
6665                )
6666                .unwrap();
6667        let observed = module.clone();
6668        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6669        let served = probes.clone();
6670        let server = tokio::spawn(async move {
6671            loop {
6672                let (mut stream, _) = listener.accept().await.unwrap();
6673                let mut request = [0; 2048];
6674                assert!(stream.read(&mut request).await.unwrap() > 0);
6675                *served
6676                    .lock()
6677                    .unwrap()
6678                    .entry(observed.status().unwrap().spawn_generation)
6679                    .or_default() += 1;
6680                let _ = stream
6681                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6682                    .await;
6683            }
6684        });
6685        timeout(Duration::from_secs(5), async {
6686            while module.state().unwrap() != ModuleState::Failed {
6687                sleep(Duration::from_millis(5)).await;
6688            }
6689        })
6690        .await
6691        .expect("sustained failures must still exhaust the health restart budget");
6692        let status = module.status().unwrap();
6693        server.abort();
6694        assert_eq!(status.restart_count, 2);
6695        assert_eq!(status.lifetime_restarts, 2);
6696        assert!(!status.process_alive);
6697        assert_eq!(
6698            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6699            vec![2, 2, 2]
6700        );
6701    }
6702
6703    #[cfg(unix)]
6704    #[tokio::test]
6705    async fn spawn_clears_the_previous_process_health() {
6706        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6707        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6708        configured.health.cadence = Duration::from_secs(3600);
6709        let module =
6710            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6711                .supervise_configured_with_health(
6712                    configured.module_spec(),
6713                    false,
6714                    configured.health,
6715                    configured.drain_timeout_ms,
6716                    configured.restart,
6717                )
6718                .unwrap();
6719        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6720            state.health.status = SupervisorHealthStatus::Unresponsive;
6721            state.health.consecutive_failures = 3;
6722            state.health.detail = Some("old process refused connections".into());
6723            state.health.metrics = Some(serde_json::json!({"old": true}));
6724            state.health.last_probe_ms = Some(123);
6725            state.health.last_action = Some("restart".into());
6726            state.health.last_action_ms = Some(124);
6727        })
6728        .unwrap();
6729        module.set_enabled(true).await.unwrap();
6730        let health = module.status().unwrap().health;
6731        module.drain().await.unwrap();
6732        assert_eq!(health.consecutive_failures, 0);
6733        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6734        assert_eq!(health.detail, None);
6735        assert_eq!(health.metrics, None);
6736        assert_eq!(health.last_probe_ms, None);
6737        assert_eq!(health.last_action.as_deref(), Some("restart"));
6738        assert_eq!(health.last_action_ms, Some(124));
6739    }
6740
6741    #[cfg(unix)]
6742    #[tokio::test]
6743    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6744        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6745        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6746        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6747        drop(unused);
6748        let configured = http_fixture(&dir, &url, 2);
6749        let module =
6750            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6751                .supervise_configured_with_health(
6752                    configured.module_spec(),
6753                    true,
6754                    configured.health,
6755                    configured.drain_timeout_ms,
6756                    configured.restart,
6757                )
6758                .unwrap();
6759        let before = module.status().unwrap().spawn_generation;
6760        timeout(Duration::from_secs(5), async {
6761            loop {
6762                let status = module.status().unwrap();
6763                if status.spawn_generation > before {
6764                    assert!(status.lifetime_restarts > 0);
6765                    break;
6766                }
6767                sleep(Duration::from_millis(5)).await;
6768            }
6769        })
6770        .await
6771        .expect("sustained HTTP refusal must trigger the health restart policy");
6772        module.drain().await.unwrap();
6773    }
6774
6775    #[cfg(unix)]
6776    #[tokio::test]
6777    async fn http_health_timeout_honours_deadline() {
6778        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6779        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6780        let server = tokio::spawn(async move {
6781            let _held = listener.accept().await.unwrap();
6782            std::future::pending::<()>().await;
6783        });
6784        let error = timeout(
6785            Duration::from_secs(1),
6786            probe_http_health(&url, Duration::from_millis(10)),
6787        )
6788        .await
6789        .expect("the probe must enforce its own deadline")
6790        .unwrap_err();
6791        server.abort();
6792        assert!(error.to_string().contains("timed out"));
6793    }
6794
6795    #[cfg(unix)]
6796    #[tokio::test]
6797    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6798        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6799        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6800        let url = format!(
6801            "http://localhost:{}/healthz",
6802            listener.local_addr().unwrap().port()
6803        );
6804        let server = tokio::spawn(async move {
6805            let (mut stream, _) = listener.accept().await.unwrap();
6806            let mut request = [0u8; 2048];
6807            assert!(stream.read(&mut request).await.unwrap() > 0);
6808            stream
6809                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6810                .await
6811                .unwrap();
6812        });
6813        // The deadline only bounds a hang. A probe that never tried the IPv6
6814        // address would be refused on 127.0.0.1 and fail at once, so a longer
6815        // deadline does not weaken the assertion; one second timed out under a
6816        // loaded parallel test run.
6817        assert_eq!(
6818            probe_http_health(&url, Duration::from_secs(10))
6819                .await
6820                .unwrap()
6821                .status,
6822            HealthStatus::Ok
6823        );
6824        server.await.unwrap();
6825    }
6826
6827    #[cfg(unix)]
6828    #[tokio::test]
6829    async fn http_health_timeout_keeps_partial_status_and_body() {
6830        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6831        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6832        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6833        let server = tokio::spawn(async move {
6834            let (mut stream, _) = listener.accept().await.unwrap();
6835            let mut request = [0u8; 2048];
6836            assert!(stream.read(&mut request).await.unwrap() > 0);
6837            stream
6838                .write_all(
6839                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6840                )
6841                .await
6842                .unwrap();
6843            std::future::pending::<()>().await;
6844        });
6845        let error = probe_http_health(&url, Duration::from_secs(1))
6846            .await
6847            .unwrap_err()
6848            .to_string();
6849        server.abort();
6850        assert!(
6851            error.contains("timed out")
6852                && error.contains("503 Unavailable")
6853                && error.contains("partial diagnostic"),
6854            "{error}"
6855        );
6856    }
6857
6858    #[cfg(unix)]
6859    #[tokio::test]
6860    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6861        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6862        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6863        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6864        let server = tokio::spawn(async move {
6865            let (mut stream, _) = listener.accept().await.unwrap();
6866            let mut request = [0u8; 2048];
6867            assert!(stream.read(&mut request).await.unwrap() > 0);
6868            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6869            let response = format!(
6870                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6871                body.len()
6872            );
6873            stream.write_all(response.as_bytes()).await.unwrap();
6874        });
6875        let error = probe_http_health(&url, Duration::from_secs(1))
6876            .await
6877            .unwrap_err()
6878            .to_string();
6879        server.await.unwrap();
6880        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6881        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6882        assert!(!error.contains("not-in-diagnostic"));
6883    }
6884
6885    #[cfg(unix)]
6886    #[tokio::test]
6887    async fn http_health_real_nats_server_monitoring() {
6888        if std::process::Command::new("nats-server")
6889            .arg("--version")
6890            .env("XDG_DATA_HOME", std::env::temp_dir())
6891            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6892            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6893            .output()
6894            .is_err()
6895        {
6896            eprintln!(
6897                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6898            );
6899            return;
6900        }
6901        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6902        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6903        let port = monitor.local_addr().unwrap().port();
6904        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6905        let client_port = client.local_addr().unwrap().port();
6906        let config = dir.join("server.conf");
6907        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6908        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6909        configured.program = PathBuf::from("nats-server");
6910        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6911        drop(monitor);
6912        drop(client);
6913        let module =
6914            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6915                .supervise_configured_with_health(
6916                    configured.module_spec(),
6917                    true,
6918                    configured.health,
6919                    configured.drain_timeout_ms,
6920                    configured.restart,
6921                )
6922                .unwrap();
6923        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6924        module.drain().await.unwrap();
6925    }
6926
6927    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6928    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6929        let supervisor =
6930            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6931        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6932        let initial = ModuleSpec {
6933            module_id: "rescan-preserves-spawn-facts".to_string(),
6934            program: PathBuf::from("/spawned/module"),
6935            args: Vec::new(),
6936            env: Vec::new(),
6937            reserved: false,
6938            reserved_prefixes: Vec::new(),
6939            protocol: ModuleProtocol::Subc,
6940            overlap: Default::default(),
6941            carry_containment: false,
6942        };
6943        let module = supervisor.supervised_module(
6944            initial.clone(),
6945            supervisor.runtime_config(),
6946            snapshot,
6947            None,
6948        );
6949        let before = module.status().unwrap();
6950        let mut replacement = initial;
6951        replacement.program = PathBuf::from("/rescanned/replacement-module");
6952
6953        module
6954            .update_configuration(replacement, HealthConfig::default(), None)
6955            .await
6956            .unwrap();
6957
6958        let after = module.status().unwrap();
6959        assert_eq!(after.pid, before.pid);
6960        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6961        assert_eq!(after.spawned_from, before.spawned_from);
6962        drop(module);
6963    }
6964}
6965
6966fn unix_ms_now() -> u64 {
6967    SystemTime::now()
6968        .duration_since(UNIX_EPOCH)
6969        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6970        .unwrap_or(0)
6971}
6972
6973async fn supervise_loop(
6974    mut spec: ModuleSpec,
6975    mut runtime: SupervisorRuntimeConfig,
6976    registry: Arc<Registry>,
6977    process_liveness: Arc<SupervisorProcessLiveness>,
6978    snapshot: SharedSnapshot,
6979    mut child: Option<SupervisedChild>,
6980    mut commands: mpsc::Receiver<SupervisorCommand>,
6981) {
6982    let mut health_probe = HealthProbeRuntime::default();
6983    // Registry writes (including embedded callers) notify this module only.
6984    // Subscribe before the first refresh; watch retains changes that arrive
6985    // while commands or probes are running. No polling fallback is needed.
6986    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6987        Ok(changes) => Some(changes),
6988        Err(err) => {
6989            // A poisoned registry cannot accept further writes, so it cannot
6990            // recover via a timer. Keep exit and command handling alive.
6991            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6992            None
6993        }
6994    };
6995    // All restart backoffs run here, including health and operator requests.
6996    // While one is pending the loop serves commands, so disable or drain can
6997    // cancel the replacement without spawning a process just to stop it.
6998    let mut pending_respawn: Option<PendingRespawn> = None;
6999    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
7000    // before anything else so a stop that interrupted a swap runs at once.
7001    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
7002    let mut previous_instance_recheck = tokio::time::interval_at(
7003        Instant::now() + Duration::from_secs(5),
7004        Duration::from_secs(5),
7005    );
7006    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
7007    loop {
7008        #[cfg(test)]
7009        {
7010            lock_snapshot(&snapshot).unwrap().actor_select = None;
7011        }
7012        #[cfg(target_os = "macos")]
7013        if let Some(active) = child.as_mut() {
7014            active.confirm_privacy_exec().await;
7015        }
7016        if let Some(scheduled) = runtime
7017            .scheduled_respawn
7018            .lock()
7019            .unwrap_or_else(|p| p.into_inner())
7020            .take()
7021        {
7022            runtime.recovery_is_daemon = scheduled.daemon_initiated;
7023            pending_respawn = Some(scheduled);
7024        }
7025        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7026            pending_respawn = None;
7027            cancel_deferred_reload(
7028                &runtime,
7029                &spec.module_id,
7030                "respawn cancelled by a supervisor command",
7031            );
7032        }
7033        if child.is_none()
7034            && pending_respawn.is_none()
7035            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7036        {
7037            cancel_deferred_reload(
7038                &runtime,
7039                &spec.module_id,
7040                "respawn cancelled before a replacement was spawned",
7041            );
7042            let mut stranded = false;
7043            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7044                state.respawn_pending = false;
7045                state.coalesced_restart_pending = false;
7046                if matches!(
7047                    state.state,
7048                    ModuleState::Restarting
7049                        | ModuleState::Starting
7050                        | ModuleState::Draining
7051                        | ModuleState::Unresponsive
7052                ) {
7053                    error!(module_id = %spec.module_id, state = ?state.state,
7054                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7055                    stranded = true;
7056                }
7057            });
7058            if stranded {
7059                fail_snapshot(
7060                    &snapshot,
7061                    Some(&spec.module_id),
7062                    None,
7063                    &runtime.failure_context(runtime.recovery_is_daemon),
7064                );
7065            }
7066        }
7067        if child.is_none()
7068            && (runtime.child_roster.is_closed()
7069                || pending_respawn.is_none()
7070                || !lock_snapshot(&snapshot)
7071                    .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7072        {
7073            containment::terminate(&runtime.child_roster.containment).await;
7074        }
7075        if let Some(command) = requeued.pop_front() {
7076            if !handle_supervisor_command(
7077                command,
7078                &mut spec,
7079                &mut runtime,
7080                &registry,
7081                &process_liveness,
7082                &snapshot,
7083                &mut child,
7084                &mut commands,
7085                &mut requeued,
7086            )
7087            .await
7088            {
7089                containment::terminate(&runtime.child_roster.containment).await;
7090                return;
7091            }
7092            if child.is_some() || !respawn_still_pending(&snapshot) {
7093                pending_respawn = None;
7094                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7095                    state.respawn_pending = false
7096                });
7097            }
7098            continue;
7099        }
7100        if child.is_some() {
7101            runtime.recovery_is_daemon = true;
7102            #[cfg(test)]
7103            {
7104                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7105            }
7106            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
7107            let active = child.as_mut().expect("child checked above");
7108            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7109                active.registration_deadline = None;
7110            }
7111            let registration_deadline = active.registration_deadline;
7112            let no_checks = runtime.health.http.is_none()
7113                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7114                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7115            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7116                state
7117                    .failed_retry
7118                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
7119            });
7120            let wake_at = match (health_probe.wake_at(), uptime_reset) {
7121                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7122                (probe, None) => probe,
7123            };
7124            let wake_at = match (wake_at, registration_deadline) {
7125                (Some(wake), Some(register)) => Some(wake.min(register)),
7126                (wake, register) => wake.or(register),
7127            };
7128            #[cfg(test)]
7129            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7130            let probe_sleep = supervisor_sleep(wake_at);
7131            tokio::pin!(probe_sleep);
7132            // A `protocol: "none"` child (a plain process that never registers
7133            // over the subc wire, such as nats-server) with no HTTP health
7134            // check has nothing to probe, so it waits only for its exit or a
7135            // supervisor command. A rescan that adds an HTTP check is a command.
7136            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7137            #[cfg(test)]
7138            {
7139                let registration_pending = wire_child
7140                    && registration_changes
7141                        .as_ref()
7142                        .is_some_and(|changes| changes.has_changed().unwrap());
7143                let mut state = lock_snapshot(&snapshot).unwrap();
7144                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7145                    generation: state.spawn_generation,
7146                    turn: state.actor_turns,
7147                    registered_connection: health_probe.registered_connection,
7148                    next_probe_at: health_probe.next_probe_at,
7149                    wake_after,
7150                });
7151            }
7152            let active_child = child.as_mut().expect("child checked above");
7153            tokio::select! {
7154                wait_result = active_child.wait() => {
7155                    if let Ok(mut state) = lock_snapshot(&snapshot) {
7156                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7157                    }
7158                    #[cfg(test)]
7159                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7160                    // Every arm below that gives up on the CHILD must keep the
7161                    // supervision task itself alive (child = None, loop
7162                    // continues into command-serving mode). Returning here
7163                    // closes the command channel, which makes the module
7164                    // permanently unrestartable in-band: a clean child exit
7165                    // of an enabled module once wedged the fleet this way
7166                    // ('supervisor command channel is closed') and required a
7167                    // full daemon restart to recover.
7168                    let exit_report = match wait_result {
7169                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7170                        Err(err) => {
7171                            active_child.drain_stderr(&spec.module_id).await;
7172                            fail_after_wait_error(
7173                                &spec.module_id,
7174                                &snapshot,
7175                                &runtime.terminal_ring,
7176                                &runtime.spawn_events,
7177                                format!("failed to wait for supervised module: {err}"),
7178                            &runtime.failure_context(true),
7179                            );
7180                            untrack_if_registration_released(
7181                                &process_liveness,
7182                                &registry,
7183                                &spec.module_id,
7184                                &snapshot,
7185                            );
7186                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7187                            child = None;
7188                            continue;
7189                        }
7190                    };
7191                    active_child.drain_stderr(&spec.module_id).await;
7192
7193                    let next = on_child_exit(
7194                        &spec,
7195                        runtime.restart_policy,
7196                        &registry,
7197                        &snapshot,
7198                        &runtime.terminal_ring,
7199                        &runtime.spawn_events,
7200                        &runtime.child_roster,
7201                        exit_report,
7202                        true,
7203                    ).await;
7204                    runtime.start_requester = None;
7205                    runtime.pending_activity_restart = false;
7206                    // The exit is recorded, so a daemon shutdown may stop
7207                    // waiting for this child (see `SupervisedChild::wait`).
7208                    let carry = matches!(next, NextAction::Restart { .. })
7209                        && active_child.spawn_failure.is_none()
7210                        && !runtime.child_roster.is_closed();
7211                    if let Err(error) = active_child.finish_exit(carry).await {
7212                        containment::terminate(&runtime.child_roster.containment).await;
7213                        fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7214                            &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7215                            &runtime.failure_context(true));
7216                        child = None;
7217                        continue;
7218                    }
7219                    match next {
7220                        NextAction::Stop { registration_released } => {
7221                            if registration_released {
7222                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7223                            }
7224                            child = None;
7225                        }
7226                        NextAction::Restart { schedule } => {
7227                            let delay = schedule.map_or(
7228                                runtime.restart_policy.delay_for_restart(0),
7229                                |schedule| schedule.delay,
7230                            );
7231                            if let Some(schedule) = schedule {
7232                                log_crash_respawn(&spec.module_id, schedule);
7233                            }
7234                            // The exited child is fully recorded at this point,
7235                            // so release it and count the backoff down in the
7236                            // command-serving branch below rather than sleeping
7237                            // here: commands cannot be received from inside this
7238                            // select arm, and an operator disable or drain that
7239                            // arrives during the backoff must cancel the pending
7240                            // respawn instead of waiting for it to spawn first.
7241                            child = None;
7242                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7243                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7244                        }
7245                    }
7246                }
7247                command = commands.recv() => {
7248                    #[cfg(test)]
7249                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7250                    let Some(command) = command else {
7251                        containment::terminate(&runtime.child_roster.containment).await;
7252                        return;
7253                    };
7254                    if !handle_supervisor_command(
7255                        command,
7256                        &mut spec,
7257                        &mut runtime,
7258                        &registry,
7259                        &process_liveness,
7260                        &snapshot,
7261                        &mut child,
7262                        &mut commands,
7263                        &mut requeued,
7264                    ).await {
7265                        containment::terminate(&runtime.child_roster.containment).await;
7266                        return;
7267                    }
7268                }
7269                _ = async {
7270                    match registration_changes.as_mut() {
7271                        Some(changes) => { let _ = changes.changed().await; }
7272                        None => std::future::pending().await,
7273                    }
7274                }, if wire_child => {
7275                    #[cfg(test)]
7276                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7277                }
7278                _ = &mut probe_sleep => {
7279                    #[cfg(test)]
7280                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7281                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7282                        // Recheck after the wake: HELLO, exit or shutdown may
7283                        // have become ready in the same executor tick.
7284                        active_child.registration_deadline = None;
7285                        if !runtime.child_roster.is_closed()
7286                            && !active_child.registered_since_spawn()
7287                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7288                        {
7289                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7290                                wait_secs = runtime.register_timeout.as_secs(),
7291                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7292                            active_child.spawn_failure = Some("registration_timeout".into());
7293                            if let Err(error) = stop_unregistered_child(active_child).await {
7294                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7295                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7296                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7297                                    &runtime.failure_context(true));
7298                                child = None;
7299                            }
7300                            // The ordinary exit handler records the child's exit,
7301                            // spends crash budget, and schedules recovery.
7302                            continue;
7303                        }
7304                    }
7305                    if health_probe.due() {
7306                        run_health_probe_cycle(
7307                            &spec,
7308                            &runtime,
7309                            &registry,
7310                            &process_liveness,
7311                            &snapshot,
7312                            &mut child,
7313                        ).await;
7314                        if child.is_none() {
7315                            runtime.start_requester = None;
7316                            runtime.pending_activity_restart = false;
7317                        }
7318                        if child.is_some() {
7319                            health_probe.schedule_next(&spec, runtime.health.cadence);
7320                        }
7321                    }
7322                }
7323            }
7324        } else if let Some(pending) = pending_respawn {
7325            tokio::select! {
7326                _ = runtime.child_roster.closed() => {
7327                    pending_respawn = None;
7328                    let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7329                        state.state = ModuleState::Stopped;
7330                        state.respawn_pending = false;
7331                    });
7332                    containment::terminate(&runtime.child_roster.containment).await;
7333                }
7334                _ = sleep_until(pending.deadline) => {
7335                    pending_respawn = None;
7336                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7337                    // A command handled below while the backoff elapsed may
7338                    // have stopped the module; never respawn past an operator's
7339                    // disable or drain.
7340                    if !respawn_still_pending(&snapshot) {
7341                        continue;
7342                    }
7343                    // The daemon began shutting down during the backoff: the
7344                    // spawn would be refused anyway, and refusing it here
7345                    // leaves the module stopped instead of reporting a
7346                    // failed restart.
7347                    if runtime.child_roster.is_closed() {
7348                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7349                            state.state = ModuleState::Stopped;
7350                        });
7351                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7352                        continue;
7353                    }
7354                    if let Err(err) = release_dead_registration(
7355                        &registry,
7356                        runtime.forwarding.as_deref(),
7357                        &snapshot,
7358                        &spec.module_id,
7359                        &runtime.failure_context(runtime.recovery_is_daemon),
7360                    ).await {
7361                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7362                        continue;
7363                    }
7364
7365                    if matches!(pending.kind, RespawnKind::Reload) {
7366                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7367                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7368                        if result.is_ok() && runtime.pending_activity_restart {
7369                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7370                        }
7371                        runtime.pending_activity_restart = false;
7372                        if let Some(reply) = reply { let _ = reply.send(result); }
7373                        continue;
7374                    }
7375                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7376                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7377                        Ok(next_child) => {
7378                            child = Some(next_child);
7379                            if runtime.pending_activity_restart {
7380                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7381                                runtime.pending_activity_restart = false;
7382                            }
7383                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7384                        }
7385                        Err(err) => {
7386                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7387                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7388                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7389                        }
7390                    }
7391                }
7392                command = commands.recv() => {
7393                    let Some(command) = command else {
7394                        containment::terminate(&runtime.child_roster.containment).await;
7395                        return;
7396                    };
7397                    if !handle_supervisor_command(
7398                        command,
7399                        &mut spec,
7400                        &mut runtime,
7401                        &registry,
7402                        &process_liveness,
7403                        &snapshot,
7404                        &mut child,
7405                        &mut commands,
7406                        &mut requeued,
7407                    ).await {
7408                        containment::terminate(&runtime.child_roster.containment).await;
7409                        return;
7410                    }
7411                    // Reconcile the pending respawn with what the command did:
7412                    // a start may already have spawned a fresh child,
7413                    // while a disable or drain moved the snapshot out of the
7414                    // state the respawn was counting down from.
7415                    if child.is_some() || !respawn_still_pending(&snapshot) {
7416                        pending_respawn = None;
7417                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7418                    }
7419                }
7420            }
7421        } else {
7422            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7423                (state.enabled && state.state == ModuleState::Failed)
7424                    .then_some(state.failed_retry.pending)
7425                    .flatten()
7426            });
7427            let command = if lock_snapshot(&snapshot)
7428                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7429            {
7430                tokio::select! {
7431                    command = commands.recv() => command,
7432                    _ = previous_instance_recheck.tick() => {
7433                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7434                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7435                            if enabled && !runtime.child_roster.is_closed() {
7436                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7437                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7438                                    Ok(next_child) => child = Some(next_child),
7439                                    Err(error) => {
7440                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7441                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7442                                    }
7443                                }
7444                            } else {
7445                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7446                                    state.waiting_for_previous_pid = None;
7447                                    state.waiting_for_previous_detail = None;
7448                                });
7449                            }
7450                        }
7451                        continue;
7452                    }
7453                }
7454            } else {
7455                tokio::select! {
7456                    biased;
7457                    command = commands.recv() => command,
7458                    _ = async {
7459                        match slow_retry {
7460                            Some(retry) => sleep_until(retry.deadline).await,
7461                            None => std::future::pending().await,
7462                        }
7463                    } => {
7464                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7465                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7466                            state.failed_retry.pending = None;
7467                        });
7468                        if runtime.child_roster.is_closed() {
7469                            continue;
7470                        }
7471                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7472                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7473                        if let Err(error) = set_child_enabled(
7474                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7475                            &runtime.failure_context(true),
7476                        ).await {
7477                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7478                        }
7479                        continue;
7480                    }
7481                }
7482            };
7483            let Some(command) = command else {
7484                containment::terminate(&runtime.child_roster.containment).await;
7485                return;
7486            };
7487            if !handle_supervisor_command(
7488                command,
7489                &mut spec,
7490                &mut runtime,
7491                &registry,
7492                &process_liveness,
7493                &snapshot,
7494                &mut child,
7495                &mut commands,
7496                &mut requeued,
7497            )
7498            .await
7499            {
7500                containment::terminate(&runtime.child_roster.containment).await;
7501                return;
7502            }
7503        }
7504    }
7505}
7506
7507// Preserve the deadline until the future is polled. Starting a relative sleep
7508// after the executor yields would restart part of the countdown.
7509async fn supervisor_sleep(wake_at: Option<Instant>) {
7510    match wake_at {
7511        Some(deadline) => sleep_until(deadline).await,
7512        None => std::future::pending().await,
7513    }
7514}
7515
7516fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7517    info!(
7518        module_id,
7519        restart_in_window = schedule.restart_in_window,
7520        delay_ms = schedule.delay.as_millis() as u64,
7521        "respawning after crash"
7522    );
7523}
7524
7525/// Whether the respawn a backoff was counting down to is still wanted. A
7526/// disable or drain handled while the backoff elapsed moves the snapshot out
7527/// of `Restarting`, and the operator's stop must win over the pending respawn,
7528/// so every sleep-then-spawn path re-validates against the live snapshot
7529/// instead of assuming the state it left behind still holds.
7530fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7531    matches!(
7532        lock_snapshot(snapshot),
7533        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7534    )
7535}
7536
7537enum NextAction {
7538    Stop {
7539        registration_released: bool,
7540    },
7541    Restart {
7542        schedule: Option<CrashRestartSchedule>,
7543    },
7544}
7545
7546#[allow(clippy::too_many_arguments)]
7547async fn handle_supervisor_command(
7548    command: SupervisorCommand,
7549    spec: &mut ModuleSpec,
7550    runtime: &mut SupervisorRuntimeConfig,
7551    registry: &Arc<Registry>,
7552    process_liveness: &SupervisorProcessLiveness,
7553    snapshot: &SharedSnapshot,
7554    child: &mut Option<SupervisedChild>,
7555    commands: &mut mpsc::Receiver<SupervisorCommand>,
7556    requeued: &mut VecDeque<SupervisorCommand>,
7557) -> bool {
7558    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7559        runtime.recovery_is_daemon = false;
7560    }
7561    if matches!(
7562        command,
7563        SupervisorCommand::Drain { .. }
7564            | SupervisorCommand::Retire { .. }
7565            | SupervisorCommand::SetEnabled { .. }
7566            | SupervisorCommand::Restart { .. }
7567    ) {
7568        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7569            state.failed_retry.cancel()
7570        });
7571    }
7572    let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7573        runtime
7574            .child_roster
7575            .previous_instance(&spec.module_id)
7576            .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7577    });
7578    let command = if let Some(error) = blocked {
7579        match command {
7580            SupervisorCommand::Restart { reply, .. }
7581            | SupervisorCommand::Reload { reply, .. }
7582            | SupervisorCommand::Swap { reply, .. } => {
7583                let _ = reply.send(Err(error));
7584                return true;
7585            }
7586            SupervisorCommand::SetEnabled {
7587                enabled: true,
7588                reply,
7589                ..
7590            } => {
7591                let _ = reply.send(Err(error));
7592                return true;
7593            }
7594            other => other,
7595        }
7596    } else {
7597        command
7598    };
7599    match command {
7600        SupervisorCommand::Drain { requester, reply } => {
7601            // A plain stop runs no forwarding drain, so nothing reaches the
7602            // module over its connection before the wait: ask by signal.
7603            let result = drain_optional_child(
7604                &spec.module_id,
7605                spec.protocol,
7606                StopNotice::NotSent,
7607                registry,
7608                runtime.forwarding.as_deref(),
7609                snapshot,
7610                &runtime.terminal_ring,
7611                &runtime.spawn_events,
7612                child,
7613                runtime.drain_timeout,
7614                ModuleState::Stopped,
7615                None,
7616                &FailureContext::operator(),
7617            )
7618            .await;
7619            containment::terminate(&runtime.child_roster.containment).await;
7620            let registration_released = result.is_ok();
7621            if registration_released {
7622                runtime.record_activity_transition(
7623                    "subc.module_stopped",
7624                    &spec.module_id,
7625                    requester,
7626                );
7627            }
7628            let _ = reply.send(result);
7629            if registration_released {
7630                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7631            }
7632            false
7633        }
7634        SupervisorCommand::Retire { requester, reply } => {
7635            let result = async {
7636                let stop_notice = begin_forwarding_drain_if_configured(
7637                    spec,
7638                    runtime,
7639                    registry,
7640                    snapshot,
7641                    None,
7642                    RouteCloseReason::Disable,
7643                )
7644                .await?;
7645                drain_optional_child(
7646                    &spec.module_id,
7647                    spec.protocol,
7648                    stop_notice,
7649                    registry,
7650                    runtime.forwarding.as_deref(),
7651                    snapshot,
7652                    &runtime.terminal_ring,
7653                    &runtime.spawn_events,
7654                    child,
7655                    runtime.drain_timeout,
7656                    ModuleState::Stopped,
7657                    None,
7658                    &FailureContext::operator(),
7659                )
7660                .await
7661            }
7662            .await;
7663            containment::terminate(&runtime.child_roster.containment).await;
7664            let registration_released = result.is_ok();
7665            if registration_released {
7666                runtime.record_activity_transition(
7667                    "subc.module_stopped",
7668                    &spec.module_id,
7669                    requester,
7670                );
7671            }
7672            let _ = reply.send(result);
7673            if registration_released {
7674                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7675            }
7676            false
7677        }
7678        SupervisorCommand::Restart {
7679            requester,
7680            drain_timeout_ms,
7681            received_at_generation,
7682            queued_at,
7683            reply,
7684        } => {
7685            // Without this line a restart that waited in the queue (behind a
7686            // health probe cycle or another command) was invisible: the log
7687            // showed only the drain timing out, minutes after the operator's call.
7688            info!(
7689                module_id = %spec.module_id,
7690                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7691                "restart command dequeued"
7692            );
7693            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7694            // caller whose own request lane rides the module being restarted: the
7695            // caller's in-flight request keeps the drain from quiescing, the drain
7696            // keeps the restart from completing, and the completion keeps the reply
7697            // from releasing the caller — so the drain always timed out and cut the
7698            // initiator with a GOODBYE, even on a healthy module. Replying once the
7699            // restart is validated lets a self-lane caller settle, which is exactly
7700            // what makes the drain succeed. Completion is observable via
7701            // supervisor.list / module status; a post-ack failure lands the module
7702            // in a visible terminal state below rather than in a reply nobody can
7703            // receive.
7704            let validation = match lock_snapshot(snapshot) {
7705                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7706                    module_id: spec.module_id.clone(),
7707                }),
7708                Ok(_) => Ok(()),
7709                Err(err) => Err(err),
7710            };
7711            let initiated = validation.is_ok();
7712            let _ = reply.send(validation);
7713            // A restart asks for a fresh process. Commands run one at a time,
7714            // so a restart queued behind another restart (two operator calls
7715            // in quick succession) is dequeued the moment the first one has
7716            // spawned its replacement -- before that process has sent HELLO.
7717            // Running it would drain and kill the process the first restart
7718            // just produced, which is the opposite of what both callers asked
7719            // for. If a process spawned after this request was received is
7720            // still supervised, the request is already satisfied. Not when the
7721            // configuration changed since that spawn: then the newer process
7722            // predates the spec this restart may exist to apply.
7723            // Generation publication can lag a child that is already serving
7724            // while its identity is recorded. Only process creation after request
7725            // receipt satisfies a restart, not publication after receipt.
7726            let satisfied_by_generation = if initiated
7727                && child
7728                    .as_ref()
7729                    .is_some_and(|child| child.spawned_at > queued_at)
7730            {
7731                lock_snapshot(snapshot).ok().and_then(|state| {
7732                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7733                })
7734            } else {
7735                None
7736            };
7737            let satisfied_by_pending = initiated
7738                && child.is_none()
7739                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7740                    // No child exists yet, so this replacement will necessarily
7741                    // be created after the queued request, unlike a published
7742                    // generation whose process may already have been serving.
7743                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7744                    if pending {
7745                        state.coalesced_restart_pending = true;
7746                    }
7747                    pending
7748                });
7749            if satisfied_by_pending {
7750                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7751            } else if let Some(generation) = satisfied_by_generation {
7752                info!(
7753                    module_id = %spec.module_id,
7754                    received_at_generation,
7755                    "restart already satisfied by generation {generation}; not restarting again"
7756                );
7757            } else if initiated {
7758                // Precedence: this restart's operator override, else the module's
7759                // configured budget (already resolved into the runtime).
7760                let drain_timeout = drain_timeout_ms
7761                    .map(Duration::from_millis)
7762                    .unwrap_or(runtime.drain_timeout);
7763                if let Err(err) = restart_child(
7764                    spec,
7765                    runtime,
7766                    registry,
7767                    process_liveness,
7768                    snapshot,
7769                    child,
7770                    drain_timeout,
7771                )
7772                .await
7773                {
7774                    warn!(
7775                        module_id = %spec.module_id,
7776                        error = %err,
7777                        "operator restart failed after initiation ack; module state carries the outcome"
7778                    );
7779                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7780                        state.state = ModuleState::Failed;
7781                        clear_current_process_facts(state);
7782                    });
7783                } else {
7784                    runtime.start_requester = requester;
7785                    runtime.pending_activity_restart = true;
7786                }
7787            }
7788            true
7789        }
7790        SupervisorCommand::Reload { requester, reply } => {
7791            let result =
7792                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7793            if result.is_ok() {
7794                runtime.start_requester = requester;
7795                runtime.pending_activity_restart = true;
7796            }
7797            if result.is_ok()
7798                && runtime
7799                    .scheduled_respawn
7800                    .lock()
7801                    .unwrap_or_else(|p| p.into_inner())
7802                    .as_ref()
7803                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7804            {
7805                *runtime
7806                    .deferred_reload_reply
7807                    .lock()
7808                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7809            } else {
7810                let _ = reply.send(result);
7811            }
7812            true
7813        }
7814        SupervisorCommand::SetEnabled {
7815            enabled,
7816            requester,
7817            reply,
7818        } => {
7819            let previous_requester = runtime.start_requester.clone();
7820            if enabled {
7821                runtime.start_requester = requester.clone();
7822            }
7823            let result = set_child_enabled(
7824                spec,
7825                runtime,
7826                registry,
7827                process_liveness,
7828                snapshot,
7829                child,
7830                enabled,
7831                &runtime.failure_context(false),
7832            )
7833            .await;
7834            if !enabled {
7835                containment::terminate(&runtime.child_roster.containment).await;
7836            }
7837            if matches!(result, Ok(true)) {
7838                runtime.pending_activity_restart = false;
7839                if enabled {
7840                    runtime.start_requester = requester;
7841                } else {
7842                    runtime.start_requester = None;
7843                    runtime.record_activity_transition(
7844                        "subc.module_disabled",
7845                        &spec.module_id,
7846                        requester,
7847                    );
7848                }
7849            } else {
7850                runtime.start_requester = previous_requester;
7851            }
7852            let _ = reply.send(result);
7853            true
7854        }
7855        SupervisorCommand::UpdateConfiguration {
7856            spec: next_spec,
7857            health,
7858            drain_timeout_ms,
7859            reply,
7860        } => {
7861            if let Some(handle) = &runtime.supervisor_handle {
7862                handle.apply_identity_configuration(&next_spec);
7863            }
7864            *spec = next_spec;
7865            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7866                state.configuration_updated_since_spawn = true;
7867            });
7868            let health_changed = runtime.health != health;
7869            runtime.health = health;
7870            // Reset the cadence and old endpoint's failure streak on a live
7871            // health-policy change rather than waiting for its old deadline.
7872            if health_changed {
7873                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7874                    state.health = ModuleHealthStatus::default();
7875                });
7876            }
7877            runtime.drain_timeout = drain_timeout_ms
7878                .map(Duration::from_millis)
7879                .unwrap_or(runtime.default_drain_timeout);
7880            *runtime
7881                .effective_drain_timeout
7882                .lock()
7883                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7884            let _ = reply.send(());
7885            true
7886        }
7887        SupervisorCommand::Swap {
7888            requester,
7889            ready_timeout,
7890            reply,
7891        } => {
7892            if spec.carry_containment
7893                || containment::current(&runtime.child_roster.containment).is_some()
7894            {
7895                let _ = reply.send(Err(SuperviseError::InvalidSpec {
7896                    reason: "a carried container cannot overlap a swap candidate".to_owned(),
7897                }));
7898                return true;
7899            }
7900            let mut swap_runtime = runtime.clone();
7901            swap_runtime.start_requester = requester;
7902            let before = lock_snapshot(snapshot)
7903                .ok()
7904                .map(|state| state.spawn_generation);
7905            let end = swap::run_swap(
7906                spec,
7907                &swap_runtime,
7908                registry,
7909                process_liveness,
7910                snapshot,
7911                child,
7912                commands,
7913                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7914                reply,
7915            )
7916            .await;
7917            if before
7918                != lock_snapshot(snapshot)
7919                    .ok()
7920                    .map(|state| state.spawn_generation)
7921            {
7922                runtime.start_requester = swap_runtime.start_requester;
7923            }
7924            requeued.extend(end.requeue);
7925            true
7926        }
7927    }
7928}
7929
7930async fn restart_child(
7931    spec: &ModuleSpec,
7932    runtime: &SupervisorRuntimeConfig,
7933    registry: &Registry,
7934    process_liveness: &SupervisorProcessLiveness,
7935    snapshot: &SharedSnapshot,
7936    child: &mut Option<SupervisedChild>,
7937    drain_timeout: Duration,
7938) -> Result<(), SuperviseError> {
7939    // Restart cycles a running module; it must not silently start a disabled one.
7940    if !lock_snapshot(snapshot)?.enabled {
7941        return Err(SuperviseError::Disabled {
7942            module_id: spec.module_id.clone(),
7943        });
7944    }
7945    let stop_notice = begin_forwarding_drain_with_timeout(
7946        spec,
7947        runtime,
7948        registry,
7949        snapshot,
7950        None,
7951        RouteCloseReason::Restart,
7952        drain_timeout,
7953    )
7954    .await?;
7955
7956    if child.is_some() {
7957        drain_optional_child(
7958            &spec.module_id,
7959            spec.protocol,
7960            stop_notice,
7961            registry,
7962            runtime.forwarding.as_deref(),
7963            snapshot,
7964            &runtime.terminal_ring,
7965            &runtime.spawn_events,
7966            child,
7967            drain_timeout,
7968            ModuleState::Restarting,
7969            Some(true),
7970            &FailureContext::operator(),
7971        )
7972        .await?;
7973    } else {
7974        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7975            state.enabled = true;
7976            state.state = ModuleState::Restarting;
7977            clear_current_process_facts(state);
7978        })?;
7979        release_dead_registration(
7980            registry,
7981            runtime.forwarding.as_deref(),
7982            snapshot,
7983            &spec.module_id,
7984            &runtime.failure_context(false),
7985        )
7986        .await?;
7987    }
7988
7989    reset_restart_count(snapshot, &spec.module_id)?;
7990    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7991    schedule_respawn(
7992        runtime,
7993        snapshot,
7994        &spec.module_id,
7995        runtime.restart_policy.backoff,
7996        RespawnKind::Spawn,
7997        false,
7998    )
7999}
8000
8001async fn reload_child(
8002    spec: &ModuleSpec,
8003    runtime: &SupervisorRuntimeConfig,
8004    registry: &Registry,
8005    process_liveness: &SupervisorProcessLiveness,
8006    snapshot: &SharedSnapshot,
8007    child: &mut Option<SupervisedChild>,
8008) -> Result<(), SuperviseError> {
8009    // Reload cycles a running module; it must not silently start a disabled one.
8010    if !lock_snapshot(snapshot)?.enabled {
8011        return Err(SuperviseError::Disabled {
8012            module_id: spec.module_id.clone(),
8013        });
8014    }
8015    let stop_notice = begin_forwarding_drain(
8016        spec,
8017        runtime,
8018        registry,
8019        snapshot,
8020        Some(true),
8021        RouteCloseReason::Reload,
8022    )
8023    .await?;
8024
8025    if child.is_some() {
8026        drain_optional_child(
8027            &spec.module_id,
8028            spec.protocol,
8029            stop_notice,
8030            registry,
8031            runtime.forwarding.as_deref(),
8032            snapshot,
8033            &runtime.terminal_ring,
8034            &runtime.spawn_events,
8035            child,
8036            runtime.drain_timeout,
8037            ModuleState::Restarting,
8038            Some(true),
8039            &FailureContext::operator(),
8040        )
8041        .await?;
8042    } else {
8043        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8044            state.enabled = true;
8045            state.state = ModuleState::Restarting;
8046            clear_current_process_facts(state);
8047        })?;
8048        release_dead_registration(
8049            registry,
8050            runtime.forwarding.as_deref(),
8051            snapshot,
8052            &spec.module_id,
8053            &runtime.failure_context(false),
8054        )
8055        .await?;
8056    }
8057
8058    reset_restart_count(snapshot, &spec.module_id)?;
8059    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8060    schedule_respawn(
8061        runtime,
8062        snapshot,
8063        &spec.module_id,
8064        runtime.restart_policy.backoff,
8065        RespawnKind::Reload,
8066        false,
8067    )
8068}
8069
8070async fn finish_reload_child(
8071    spec: &ModuleSpec,
8072    runtime: &SupervisorRuntimeConfig,
8073    registry: &Registry,
8074    process_liveness: &SupervisorProcessLiveness,
8075    snapshot: &SharedSnapshot,
8076    child: &mut Option<SupervisedChild>,
8077) -> Result<(), SuperviseError> {
8078    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8079    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8080        Ok(next_child) => next_child,
8081        Err(err) => {
8082            return handle_reload_spawn_failure(
8083                spec,
8084                runtime,
8085                process_liveness,
8086                snapshot,
8087                child,
8088                format!("new child failed to spawn: {err}"),
8089            )
8090            .await;
8091        }
8092    };
8093    *child = Some(next_child);
8094
8095    let wait_outcome = {
8096        let active_child = child.as_mut().expect("new reload child was just stored");
8097        wait_for_registration_after_reload(
8098            registry,
8099            &spec.module_id,
8100            snapshot,
8101            active_child,
8102            REGISTRY_RELEASE_TIMEOUT,
8103        )
8104        .await?
8105    };
8106
8107    match wait_outcome {
8108        RegistrationWaitOutcome::Registered => {
8109            child
8110                .as_mut()
8111                .expect("registered reload child")
8112                .registration_deadline = None;
8113            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8114            Ok(())
8115        }
8116        RegistrationWaitOutcome::Exited(exit_report) => {
8117            if let Some(active_child) = child.as_mut() {
8118                active_child.drain_stderr(&spec.module_id).await;
8119            }
8120            // Keep the exited child's roster guard until its terminal is written.
8121            // Shutdown waits on that guard, not on the child Option used for respawn.
8122            let mut exited_child = child.take().expect("exited reload child is still stored");
8123            #[cfg(test)]
8124            if let Some(gate) = &runtime.test_reload_exit_record_gate {
8125                gate.reached.notify_one();
8126                gate.resume.notified().await;
8127            }
8128            let result = handle_reload_child_registration_failure(
8129                spec,
8130                runtime,
8131                registry,
8132                process_liveness,
8133                snapshot,
8134                child,
8135                ReloadRegistrationFailure {
8136                    exit_report: registration_failure_exit_report(exit_report),
8137                    reason: exited_child
8138                        .spawn_failure
8139                        .clone()
8140                        .unwrap_or_else(|| "new child exited before registering".to_string()),
8141                },
8142            )
8143            .await;
8144            exited_child
8145                .finish_exit(false)
8146                .await
8147                .map_err(|source| SuperviseError::Wait {
8148                    module_id: spec.module_id.clone(),
8149                    source,
8150                })?;
8151            result
8152        }
8153        RegistrationWaitOutcome::TimedOut => {
8154            let mut timed_out_child = child
8155                .take()
8156                .expect("timed-out reload child is still running");
8157            timed_out_child
8158                .start_kill()
8159                .map_err(|source| SuperviseError::Kill {
8160                    module_id: spec.module_id.clone(),
8161                    source,
8162                })?;
8163            let status = timed_out_child
8164                .wait()
8165                .await
8166                .map_err(|source| SuperviseError::Wait {
8167                    module_id: spec.module_id.clone(),
8168                    source,
8169                })?;
8170            timed_out_child.drain_stderr(&spec.module_id).await;
8171            let result = handle_reload_child_registration_failure(
8172                spec,
8173                runtime,
8174                registry,
8175                process_liveness,
8176                snapshot,
8177                child,
8178                ReloadRegistrationFailure {
8179                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8180                        snapshot,
8181                        &timed_out_child,
8182                        &status,
8183                    )),
8184                    reason: format!(
8185                        "new child did not register within {:?}",
8186                        REGISTRY_RELEASE_TIMEOUT
8187                    ),
8188                },
8189            )
8190            .await;
8191            timed_out_child
8192                .finish_exit(false)
8193                .await
8194                .map_err(|source| SuperviseError::Wait {
8195                    module_id: spec.module_id.clone(),
8196                    source,
8197                })?;
8198            result
8199        }
8200    }
8201}
8202
8203#[allow(clippy::too_many_arguments)]
8204async fn set_child_enabled(
8205    spec: &ModuleSpec,
8206    runtime: &SupervisorRuntimeConfig,
8207    registry: &Registry,
8208    process_liveness: &SupervisorProcessLiveness,
8209    snapshot: &SharedSnapshot,
8210    child: &mut Option<SupervisedChild>,
8211    enabled: bool,
8212    failure: &FailureContext,
8213) -> Result<bool, SuperviseError> {
8214    let (current_enabled, current_state, respawn_pending) = {
8215        let state = lock_snapshot(snapshot)?;
8216        (state.enabled, state.state, state.respawn_pending)
8217    };
8218    // `start` (enable on an already-enabled module) heals TERMINAL states instead
8219    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
8220    // clean (Stopped) has no live process and no other in-band recovery — the
8221    // operator's start is the explicit recovery act and resets the budget. Without
8222    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
8223    // which the 2026-07-14 aft outage proved is a trap when the failed module is
8224    // the one providing every agent's shell.
8225    let revive_terminal = enabled
8226        && current_enabled
8227        && child.is_none()
8228        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8229            || (current_state == ModuleState::Restarting && !respawn_pending));
8230    if current_enabled == enabled && !revive_terminal {
8231        return Ok(false);
8232    }
8233
8234    if enabled {
8235        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8236            state.enabled = true;
8237            state.state = ModuleState::Starting;
8238            clear_current_process_facts(state);
8239        })?;
8240        #[cfg(test)]
8241        if runtime.test_seed_stale_facts_before_enable_spawn {
8242            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8243                state.process_alive = true;
8244                state.pid = Some(41);
8245                state.spawned_at_ms = Some(42);
8246                state.spawned_from = Some(PathBuf::from("/spawned/module"));
8247                state.spawned_file_identity = Some(SpawnedFileIdentity {
8248                    device: 43,
8249                    inode: 44,
8250                    #[cfg(windows)]
8251                    windows_image: None,
8252                });
8253            })?;
8254        }
8255        release_dead_registration(
8256            registry,
8257            runtime.forwarding.as_deref(),
8258            snapshot,
8259            &spec.module_id,
8260            failure,
8261        )
8262        .await?;
8263        reset_restart_count(snapshot, &spec.module_id)?;
8264        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8265        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8266            Ok(next_child) => next_child,
8267            Err(err) => {
8268                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8269                process_liveness.untrack_if_current(&spec.module_id, snapshot);
8270                return Err(err);
8271            }
8272        };
8273        *child = Some(next_child);
8274        debug!(module_id = %spec.module_id, "supervised module enabled");
8275        Ok(true)
8276    } else {
8277        let stop_notice = begin_forwarding_drain_if_configured(
8278            spec,
8279            runtime,
8280            registry,
8281            snapshot,
8282            Some(false),
8283            RouteCloseReason::Disable,
8284        )
8285        .await?;
8286        drain_optional_child(
8287            &spec.module_id,
8288            spec.protocol,
8289            stop_notice,
8290            registry,
8291            runtime.forwarding.as_deref(),
8292            snapshot,
8293            &runtime.terminal_ring,
8294            &runtime.spawn_events,
8295            child,
8296            runtime.drain_timeout,
8297            ModuleState::Disabled,
8298            Some(false),
8299            failure,
8300        )
8301        .await?;
8302        debug!(module_id = %spec.module_id, "supervised module disabled");
8303        Ok(true)
8304    }
8305}
8306
8307#[allow(clippy::too_many_arguments)]
8308async fn on_child_exit(
8309    spec: &ModuleSpec,
8310    policy: RestartPolicy,
8311    registry: &Registry,
8312    snapshot: &SharedSnapshot,
8313    terminal_ring: &Arc<Mutex<TerminalRing>>,
8314    spawn_events: &SpawnEventFeed,
8315    roster: &ChildRoster,
8316    exit_report: ExitReport,
8317    daemon_initiated: bool,
8318) -> NextAction {
8319    // Once the daemon has begun shutting down, no exit is a crash to recover
8320    // from: the module is exiting because the daemon is going away (EOF on its
8321    // connection, or a service manager signalling the whole cgroup). Record it
8322    // as such and never schedule a respawn, which would only start a process
8323    // for the shutdown to end again.
8324    if roster.is_closed() {
8325        return on_child_exit_during_daemon_shutdown(
8326            spec,
8327            registry,
8328            snapshot,
8329            terminal_ring,
8330            spawn_events,
8331            exit_report,
8332        )
8333        .await;
8334    }
8335    // Deliberate stops the supervisor asks for (operator stop, disable,
8336    // restart, reload, swap, a health restart, a drain that runs out of budget)
8337    // takes the child out of the supervise loop and reaps it in
8338    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8339    // that reaches this point is either unrequested or a first-HELLO timeout,
8340    // which is deliberately classified as a crash even after a clean TERM exit.
8341    //
8342    // For a subc-wire module a clean exit is still a stop: those modules are
8343    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8344    // crash, and exiting 0 is a deliberate choice the module made. A
8345    // `protocol: "none"` module is a stock program we cannot change, and many
8346    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8347    // stop would leave the module down for good after any stray signal, so it
8348    // goes through the crash path instead: it spends restart budget, respawns
8349    // with the crash backoff, and ends `failed` when the budget runs out.
8350    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8351        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8352    match exit_report.kind {
8353        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8354            info!(
8355                module_id = %spec.module_id,
8356                exit_code = ?exit_report.code,
8357                exit_signal = ?exit_report.signal,
8358                "supervised module exited cleanly"
8359            );
8360            if let Err(err) = publish_after_terminal_record(
8361                &spec.module_id,
8362                || {
8363                    record_terminal(
8364                        &spec.module_id,
8365                        terminal_ring,
8366                        spawn_events,
8367                        &exit_report,
8368                        TerminalDisposition::Stopped,
8369                    )
8370                },
8371                || {
8372                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8373                        state.state = ModuleState::Stopped;
8374                        clear_current_process_facts(state);
8375                        state.last_exit = Some(exit_report.clone());
8376                    })
8377                },
8378            ) {
8379                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8380            }
8381            let registration_released = match wait_for_registration_release(
8382                registry,
8383                &spec.module_id,
8384                REGISTRY_RELEASE_TIMEOUT,
8385            )
8386            .await
8387            {
8388                Ok(()) => true,
8389                Err(err) => {
8390                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8391                    false
8392                }
8393            };
8394            NextAction::Stop {
8395                registration_released,
8396            }
8397        }
8398        ExitKind::Clean | ExitKind::Crash => {
8399            if unrequested_clean_exit_of_protocol_none {
8400                warn!(
8401                    module_id = %spec.module_id,
8402                    exit_code = ?exit_report.code,
8403                    exit_signal = ?exit_report.signal,
8404                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8405                );
8406            } else {
8407                warn!(
8408                    module_id = %spec.module_id,
8409                    exit_code = ?exit_report.code,
8410                    exit_signal = ?exit_report.signal,
8411                    "supervised module exited abnormally (crash)"
8412                );
8413            }
8414            let mut restart_schedule = None;
8415            let mut disposition = TerminalDisposition::Disabled;
8416            let mut final_state = ModuleState::Disabled;
8417            // Set only when the budget is what stopped the module, so the
8418            // terminal record says which limit was hit rather than leaving
8419            // `failed` to be read as "crashed once, badly".
8420            let mut disposition_detail = lock_snapshot(snapshot)
8421                .ok()
8422                .and_then(|mut state| state.spawn_failure.take());
8423            let now = Instant::now();
8424            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8425                if state.enabled {
8426                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8427                        // Reserving the last slot exhausts the budget for the
8428                        // replacement, not for this exit. Publish its scheduled
8429                        // recovery atomically with the reservation so connection
8430                        // cleanup sees it while terminal recording still holds
8431                        // the visible state at Running.
8432                        state.respawn_pending = true;
8433                        final_state = ModuleState::Restarting;
8434                        restart_schedule = Some(schedule);
8435                        disposition = TerminalDisposition::Restarting;
8436                    } else {
8437                        final_state = ModuleState::Failed;
8438                        disposition = TerminalDisposition::Failed;
8439                        let budget = policy.budget_exhausted_detail();
8440                        disposition_detail =
8441                            Some(disposition_detail.take().map_or_else(
8442                                || budget.clone(),
8443                                |cause| format!("{cause}; {budget}"),
8444                            ));
8445                    }
8446                } else {
8447                    disposition = TerminalDisposition::Disabled;
8448                }
8449            }) {
8450                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8451                return NextAction::Stop {
8452                    registration_released: false,
8453                };
8454            }
8455            if disposition == TerminalDisposition::Failed {
8456                // The window is in the message, not only in the fields: this line
8457                // is read in a scrollback where a bare `max_restarts=3` reads as a
8458                // lifetime cap and sends the operator looking for three crashes
8459                // that never happened together.
8460                error!(
8461                    module_id = %spec.module_id,
8462                    max_restarts = policy.max_restarts,
8463                    window_secs = policy.window.as_secs(),
8464                    "module stopped: {}",
8465                    policy.budget_exhausted_detail()
8466                );
8467            }
8468            if let Err(err) = publish_after_terminal_record(
8469                &spec.module_id,
8470                || {
8471                    record_terminal_with_detail(
8472                        &spec.module_id,
8473                        terminal_ring,
8474                        spawn_events,
8475                        &exit_report,
8476                        disposition,
8477                        disposition_detail,
8478                    )
8479                },
8480                || {
8481                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8482                        state.state = final_state;
8483                        clear_current_process_facts(state);
8484                        if final_state == ModuleState::Failed
8485                            && daemon_initiated
8486                            && !roster.is_closed()
8487                        {
8488                            state.failed_retry.schedule(&spec.module_id);
8489                        }
8490                        state.last_exit = Some(exit_report.clone());
8491                    })
8492                },
8493            ) {
8494                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8495            }
8496
8497            if let Some(schedule) = restart_schedule {
8498                NextAction::Restart {
8499                    schedule: Some(schedule),
8500                }
8501            } else {
8502                let registration_released = match wait_for_registration_release(
8503                    registry,
8504                    &spec.module_id,
8505                    REGISTRY_RELEASE_TIMEOUT,
8506                )
8507                .await
8508                {
8509                    Ok(()) => true,
8510                    Err(err) => {
8511                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8512                        false
8513                    }
8514                };
8515                NextAction::Stop {
8516                    registration_released,
8517                }
8518            }
8519        }
8520        ExitKind::DeliberateSeverance => {
8521            warn!(
8522                module_id = %spec.module_id,
8523                exit_code = ?exit_report.code,
8524                exit_signal = ?exit_report.signal,
8525                "supervised module exited after deliberate connection severance"
8526            );
8527            let should_restart = match lock_snapshot(snapshot) {
8528                Ok(state) => state.enabled,
8529                Err(err) => {
8530                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8531                    return NextAction::Stop {
8532                        registration_released: false,
8533                    };
8534                }
8535            };
8536            let final_state = if should_restart {
8537                ModuleState::Restarting
8538            } else {
8539                ModuleState::Disabled
8540            };
8541            if let Err(err) = publish_after_terminal_record(
8542                &spec.module_id,
8543                || {
8544                    record_terminal(
8545                        &spec.module_id,
8546                        terminal_ring,
8547                        spawn_events,
8548                        &exit_report,
8549                        terminal_disposition(final_state),
8550                    )
8551                },
8552                || {
8553                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8554                        state.state = final_state;
8555                        clear_current_process_facts(state);
8556                        state.last_exit = Some(exit_report.clone());
8557                        state.lifetime_restarts += 1;
8558                    })
8559                },
8560            ) {
8561                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8562                return NextAction::Stop {
8563                    registration_released: false,
8564                };
8565            }
8566
8567            if should_restart {
8568                NextAction::Restart { schedule: None }
8569            } else {
8570                let registration_released = match wait_for_registration_release(
8571                    registry,
8572                    &spec.module_id,
8573                    REGISTRY_RELEASE_TIMEOUT,
8574                )
8575                .await
8576                {
8577                    Ok(()) => true,
8578                    Err(err) => {
8579                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8580                        false
8581                    }
8582                };
8583                NextAction::Stop {
8584                    registration_released,
8585                }
8586            }
8587        }
8588    }
8589}
8590
8591async fn on_child_exit_during_daemon_shutdown(
8592    spec: &ModuleSpec,
8593    registry: &Registry,
8594    snapshot: &SharedSnapshot,
8595    terminal_ring: &Arc<Mutex<TerminalRing>>,
8596    spawn_events: &SpawnEventFeed,
8597    exit_report: ExitReport,
8598) -> NextAction {
8599    info!(
8600        module_id = %spec.module_id,
8601        exit_code = ?exit_report.code,
8602        exit_signal = ?exit_report.signal,
8603        exit_kind = ?exit_report.kind,
8604        "supervised module exited during daemon shutdown; not restarting it"
8605    );
8606    if let Err(err) = publish_after_terminal_record(
8607        &spec.module_id,
8608        || {
8609            record_terminal(
8610                &spec.module_id,
8611                terminal_ring,
8612                spawn_events,
8613                &exit_report,
8614                TerminalDisposition::DaemonShutdown,
8615            )
8616        },
8617        || {
8618            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8619                state.state = ModuleState::Stopped;
8620                clear_current_process_facts(state);
8621                state.last_exit = Some(exit_report.clone());
8622            })
8623        },
8624    ) {
8625        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8626    }
8627    let registration_released =
8628        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8629            .await
8630            .is_ok();
8631    NextAction::Stop {
8632        registration_released,
8633    }
8634}
8635
8636// Keep the status lock out of terminal I/O. Readers may observe the previous
8637// state while recording is in progress, but never a completed exit without its
8638// history. Journal errors are logged by the journal and retained in the ring;
8639// a recording panic still publishes the final state before resuming the unwind.
8640fn publish_after_terminal_record<T>(
8641    module_id: &str,
8642    record: impl FnOnce(),
8643    publish: impl FnOnce() -> T,
8644) -> T {
8645    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8646    let published = publish();
8647    if let Err(panic) = recorded {
8648        error!(
8649            module_id,
8650            "terminal recording panicked; published final state before resuming unwind"
8651        );
8652        std::panic::resume_unwind(panic);
8653    }
8654    published
8655}
8656
8657fn fail_after_wait_error(
8658    module_id: &str,
8659    snapshot: &SharedSnapshot,
8660    terminal_ring: &Arc<Mutex<TerminalRing>>,
8661    spawn_events: &SpawnEventFeed,
8662    detail: String,
8663    failure: &FailureContext,
8664) {
8665    let report = wait_error_exit_report();
8666    publish_after_terminal_record(
8667        module_id,
8668        || {
8669            record_terminal_with_detail(
8670                module_id,
8671                terminal_ring,
8672                spawn_events,
8673                &report,
8674                TerminalDisposition::Failed,
8675                Some(detail),
8676            )
8677        },
8678        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8679    );
8680}
8681
8682#[cfg(test)]
8683fn record_wait_error_terminal(
8684    module_id: &str,
8685    terminal_ring: &Arc<Mutex<TerminalRing>>,
8686    spawn_events: &SpawnEventFeed,
8687) {
8688    record_terminal(
8689        module_id,
8690        terminal_ring,
8691        spawn_events,
8692        &wait_error_exit_report(),
8693        TerminalDisposition::Failed,
8694    );
8695}
8696
8697fn record_terminal(
8698    module_id: &str,
8699    terminal_ring: &Arc<Mutex<TerminalRing>>,
8700    spawn_events: &SpawnEventFeed,
8701    exit_report: &ExitReport,
8702    disposition: TerminalDisposition,
8703) {
8704    record_terminal_with_detail(
8705        module_id,
8706        terminal_ring,
8707        spawn_events,
8708        exit_report,
8709        disposition,
8710        None,
8711    );
8712}
8713
8714/// The ring lock is held only to capture the read (see
8715/// `TerminalJournal::capture_read`), so this module's exits keep recording
8716/// while the journal files are read. Blocking: it reads files.
8717fn durable_terminal_history_of(
8718    terminal_ring: &Mutex<TerminalRing>,
8719    module_id: &str,
8720) -> subc_control::TerminalHistory {
8721    let read = terminal_ring
8722        .lock()
8723        .unwrap_or_else(|p| p.into_inner())
8724        .capture_durable_history();
8725    read.read(module_id)
8726}
8727
8728fn record_terminal_with_detail(
8729    module_id: &str,
8730    terminal_ring: &Arc<Mutex<TerminalRing>>,
8731    spawn_events: &SpawnEventFeed,
8732    exit_report: &ExitReport,
8733    disposition: TerminalDisposition,
8734    disposition_detail: Option<String>,
8735) {
8736    #[cfg(test)]
8737    terminal_history_tests::before_terminal_record(spawn_events);
8738    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8739    let record = TerminalRecord {
8740        exit_code: exit_report.code,
8741        exit_signal: exit_report.signal,
8742        at_ms: exit_report.at_ms,
8743        disposition,
8744        exit_kind: exit_report.kind.into(),
8745        disposition_detail,
8746    };
8747    terminal_ring
8748        .lock()
8749        .unwrap_or_else(|poisoned| poisoned.into_inner())
8750        .record_exit(module_id, record);
8751}
8752
8753fn untrack_if_registration_released(
8754    process_liveness: &SupervisorProcessLiveness,
8755    registry: &Registry,
8756    module_id: &str,
8757    snapshot: &SharedSnapshot,
8758) {
8759    match registry.get_module(module_id) {
8760        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8761        Ok(Some(_)) => {}
8762        Err(err) => {
8763            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8764        }
8765    }
8766}
8767
8768/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8769/// then apply the module's configured entries minus daemon-private capture keys.
8770///
8771/// Separated from `spawn_child` only so it can be asserted without spawning a
8772/// process — a duplicate of this logic in a test would pass while the real one
8773/// drifted, which is the defect class this function exists to avoid.
8774/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8775/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8776/// either and the argument would stop a stock binary from starting at all.
8777/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8778///
8779/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8780/// through [`apply_wire_spawn_args_for_role`].
8781#[cfg(test)]
8782fn apply_wire_spawn_args(
8783    command: &mut Command,
8784    spec: &ModuleSpec,
8785    connection_file_path: Option<&std::path::Path>,
8786    handle: Option<&SupervisorHandle>,
8787) -> Result<Option<NonceHandoff>, SuperviseError> {
8788    apply_wire_spawn_args_for_role(
8789        command,
8790        spec,
8791        connection_file_path,
8792        handle,
8793        SpawnRole::Plain,
8794    )
8795}
8796
8797/// The read end of a spawn's launch-nonce pipe, prepared by
8798/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8799/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8800/// server before spawn and authenticates its client after the child exists.
8801#[cfg(unix)]
8802type NonceHandoff = subc_os::LaunchNonceHandoff;
8803#[cfg(windows)]
8804type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8805#[cfg(not(any(unix, windows)))]
8806type NonceHandoff = std::convert::Infallible;
8807
8808/// Create the launch nonce that a new module process must present when it
8809/// registers or opens a consumer connection, including a replacement process
8810/// started alongside a still-serving module.
8811///
8812/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8813/// a separate candidate token so the still-serving incumbent and its consumers
8814/// keep their nonce. Both records are installed before the process exists, so
8815/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8816///
8817/// On Unix the nonce is delivered only through a pipe. It is written into
8818/// a pipe whose read end the child gets as descriptor 3, named by
8819/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8820/// process of the same user cannot read it with `ps eww`. That handoff is
8821/// returned rather than installed here, because installing it replaces
8822/// whatever the child has at descriptor 3 and so must be the last pre-exec
8823/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8824/// the Linux callback that writes the process into its cgroup. Only after that
8825/// does `handoff.install_last` register the descriptor-3 replacement, so it
8826/// cannot close a descriptor the cgroup callback still needs.
8827/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8828fn apply_wire_spawn_args_for_role(
8829    command: &mut Command,
8830    spec: &ModuleSpec,
8831    connection_file_path: Option<&std::path::Path>,
8832    handle: Option<&SupervisorHandle>,
8833    role: SpawnRole,
8834) -> Result<Option<NonceHandoff>, SuperviseError> {
8835    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8836    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8837    // included: a daemon started from a module's process tree inherits it,
8838    // and passing it on would point the child at a descriptor it does not
8839    // have.
8840    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8841    #[cfg(windows)]
8842    {
8843        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8844        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8845    }
8846    // Remove inherited or configured copies too: withholding must mean absent.
8847    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8848    if spec.protocol == ModuleProtocol::None {
8849        return Ok(None);
8850    }
8851    if let Some(connection_file_path) = connection_file_path {
8852        command.arg(SUBC_ARG).arg(connection_file_path);
8853    }
8854
8855    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8856    // route.open attestation. Reserved modules additionally use the same nonce
8857    // for HELLO id-squatting protection. A respawn rotates both records.
8858    let nonce = generate_launch_nonce()?;
8859    if let Some(handle) = handle {
8860        match role {
8861            SpawnRole::Plain => {
8862                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8863                if spec.reserved {
8864                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8865                }
8866            }
8867            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8868        }
8869    }
8870    #[cfg(unix)]
8871    let handoff = {
8872        let handoff =
8873            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8874                program: spec.program.clone(),
8875                source,
8876                cgroup_path: None,
8877            })?;
8878        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8879        Some(handoff)
8880    };
8881    #[cfg(windows)]
8882    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8883    #[cfg(not(any(unix, windows)))]
8884    let handoff = None;
8885    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8886    // until the daemon's per-spawn delivery records show that every running
8887    // Windows module has read its secret from the named pipe.
8888    #[cfg(not(unix))]
8889    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8890    Ok(handoff)
8891}
8892
8893fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8894    command.env_remove(CK_LOG_ENV);
8895    // The spawn role is the supervisor's to set, and only on a swap candidate
8896    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8897    // it, is what makes it absent on a plain spawn: the daemon's own
8898    // environment could carry it, and so could a spec built outside daemon
8899    // config (config refuses it as an `env` key). A module reading it on a
8900    // plain restart would pick the long swap budget and leave callers waiting.
8901    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8902    for (key, value) in &spec.env {
8903        // cortexkit-log currently exposes retention only as a Rust struct, not
8904        // environment names. These values are daemon-private sink metadata and
8905        // must never become a public child-process contract by being inherited.
8906        if matches!(
8907            key.as_str(),
8908            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8909        ) || key == SUBC_SPAWN_ROLE_ENV
8910        {
8911            continue;
8912        }
8913        command.env(key, value);
8914    }
8915}
8916
8917#[cfg(windows)]
8918fn prepare_windows_nonce_pipe(
8919    command: &mut Command,
8920    spec: &ModuleSpec,
8921    nonce: &str,
8922) -> Option<NonceHandoff> {
8923    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8924        Ok(handoff) => {
8925            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8926            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8927            // still register until every module reads the pipe. A reader without
8928            // this variable set to env returns the pipe error instead of using
8929            // an inherited or configured environment copy.
8930            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8931            Some(handoff)
8932        }
8933        Err(error) => {
8934            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8935            None
8936        }
8937    }
8938}
8939
8940#[cfg(windows)]
8941fn start_windows_nonce_pipe(
8942    handoff: Option<NonceHandoff>,
8943    pid: u32,
8944    spec: &ModuleSpec,
8945) -> Option<subc_os::LaunchNoncePipeDelivery> {
8946    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8947    // same maximum wait used for registration during a reload. Older modules
8948    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8949    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8950        Ok(delivery) => Some(delivery),
8951        Err(error) => {
8952            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8953            None
8954        }
8955    })
8956}
8957
8958fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8959    #[cfg(windows)]
8960    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8961        return Some(
8962            if snapshot
8963                .pipe_consumed
8964                .as_ref()
8965                .is_some_and(|consumed| consumed.get().is_some())
8966            {
8967                "pipe"
8968            } else {
8969                "env"
8970            }
8971            .to_owned(),
8972        );
8973    }
8974    let _ = snapshot;
8975    None
8976}
8977
8978/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8979/// of a blue/green swap.
8980#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8981enum SpawnRole {
8982    Plain,
8983    SwapCandidate,
8984}
8985
8986/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8987/// `apply_child_env` has already removed the variable for every spawn.
8988fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8989    if role == SpawnRole::SwapCandidate {
8990        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8991    }
8992}
8993
8994fn previous_instance_error(
8995    spec: &ModuleSpec,
8996    pid: u32,
8997    reason: crate::live_children::RetainedReason,
8998) -> SuperviseError {
8999    SuperviseError::Spawn {
9000        program: spec.program.clone(),
9001        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
9002        cgroup_path: None,
9003    }
9004}
9005
9006fn carries_containment(configured: bool, has_container: bool) -> bool {
9007    configured || has_container
9008}
9009
9010fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
9011    if !carries_containment(
9012        spec.carry_containment,
9013        containment::current(&roster.containment).is_some(),
9014    ) {
9015        return None;
9016    }
9017    Some(SuperviseError::Spawn {
9018        program: spec.program.clone(),
9019        source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9020        cgroup_path: None,
9021    })
9022}
9023
9024fn spawn_child(
9025    spec: &ModuleSpec,
9026    connection_file_path: Option<&std::path::Path>,
9027    handle: Option<&SupervisorHandle>,
9028    ring: &Arc<Mutex<StderrRing>>,
9029    capture_logs_dir: Option<&std::path::Path>,
9030    roster: &ChildRoster,
9031    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9032) -> Result<SupervisedChild, SuperviseError> {
9033    #[cfg(all(test, unix))]
9034    if let Some(source) = containment_tests::join_failure(spec, roster) {
9035        containment::terminate_blocking(&roster.containment);
9036        return Err(SuperviseError::Spawn {
9037            program: spec.program.clone(),
9038            source,
9039            cgroup_path: None,
9040        });
9041    }
9042    let result = spawn_child_in_slot(
9043        spec,
9044        connection_file_path,
9045        handle,
9046        ring,
9047        capture_logs_dir,
9048        roster,
9049        #[cfg(target_os = "linux")]
9050        cgroup_placement,
9051        SpawnRole::Plain,
9052        false,
9053    );
9054    if result.is_err() {
9055        if roster.admission_pending(&spec.module_id) {
9056            containment::terminate_deferred(&roster.containment);
9057        } else {
9058            containment::terminate_blocking(&roster.containment);
9059        }
9060    }
9061    result
9062}
9063
9064/// Spawn one process of `spec` into a slot.
9065///
9066/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
9067/// A swap candidate needs a different cgroup from the process it is replacing,
9068/// which is still alive: in the same cgroup the two would be one kill domain,
9069/// and killing a failed candidate could take the incumbent with it.
9070///
9071/// The stderr capture file is `<module_id>.stderr.log` for every process of
9072/// the module, whichever slot it is in, because that is the one file
9073/// `ck module logs` reads. During a swap's overlap both processes append to it;
9074/// the daemon writes whole lines, so the two interleave by line, which is also
9075/// the merged view an operator wants while a swap runs.
9076#[allow(clippy::too_many_arguments)]
9077fn spawn_child_in_slot(
9078    spec: &ModuleSpec,
9079    connection_file_path: Option<&std::path::Path>,
9080    handle: Option<&SupervisorHandle>,
9081    ring: &Arc<Mutex<StderrRing>>,
9082    capture_logs_dir: Option<&std::path::Path>,
9083    roster: &ChildRoster,
9084    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9085    role: SpawnRole,
9086    alternate_slot: bool,
9087) -> Result<SupervisedChild, SuperviseError> {
9088    if let Some(error) = recovery_block_error(spec, roster) {
9089        return Err(error);
9090    }
9091    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9092        return Err(previous_instance_error(spec, pid, reason));
9093    }
9094    if roster.is_closed() {
9095        return Err(SuperviseError::Spawn {
9096            program: spec.program.clone(),
9097            source: io::Error::other("the daemon is shutting down; not starting a new process"),
9098            cgroup_path: None,
9099        });
9100    }
9101    let carried = containment::current(&roster.containment);
9102    let carrying = carries_containment(spec.carry_containment, carried.is_some());
9103    if role != SpawnRole::Plain && carrying {
9104        return Err(SuperviseError::InvalidSpec {
9105            reason: "a carried container cannot overlap a swap candidate".to_owned(),
9106        });
9107    }
9108    #[cfg(unix)]
9109    let launch_identity = if carrying && carried.is_none() {
9110        Some(
9111            containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9112                program: spec.program.clone(),
9113                source,
9114                cgroup_path: None,
9115            })?,
9116        )
9117    } else {
9118        None
9119    };
9120    #[cfg(target_os = "linux")]
9121    let cgroup_placement = match &carried {
9122        Some(container) => container.cgroup().map(|(placement, _)| placement),
9123        None => cgroup_placement,
9124    };
9125    #[cfg(unix)]
9126    let group_target = if let Some(container) = carried
9127        .as_ref()
9128        .filter(|container| container.group().is_some())
9129    {
9130        containment::GroupTarget::Pinned(Arc::clone(container))
9131    } else {
9132        #[cfg(target_os = "linux")]
9133        let cgroup_container = carrying && cgroup_placement.is_some();
9134        #[cfg(not(target_os = "linux"))]
9135        let cgroup_container = false;
9136        if cgroup_container {
9137            containment::GroupTarget::NoGroup
9138        } else {
9139            containment::GroupTarget::OwnLeader
9140        }
9141    };
9142    #[cfg(target_os = "linux")]
9143    let cgroup_name = {
9144        // Slot names alone are not kill domains: a retired incumbent may still
9145        // be draining when a later enable/restart spawns into the same slot.
9146        // Decimal entropy keeps the suffix unambiguous; Placement performs
9147        // the module-id escaping and constructs the filesystem path.
9148        if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9149            name.to_owned()
9150        } else if cgroup_placement.is_none() {
9151            swap::cgroup_name(&spec.module_id, alternate_slot)
9152        } else {
9153            let nonce = generate_launch_nonce()?;
9154            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9155            // Leave room for byte escaping and the suffix under NAME_MAX. The
9156            // label is only for humans; the nonce identifies the kill domain.
9157            let mut end = spec.module_id.len().min(64);
9158            while !spec.module_id.is_char_boundary(end) {
9159                end -= 1;
9160            }
9161            format!(
9162                "{}_{suffix}",
9163                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9164            )
9165        }
9166    };
9167    #[cfg(not(target_os = "linux"))]
9168    let _ = alternate_slot;
9169    #[cfg(target_os = "macos")]
9170    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9171    #[cfg(windows)]
9172    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9173        program: spec.program.clone(),
9174        source,
9175        cgroup_path: None,
9176    })?;
9177    #[cfg(windows)]
9178    let mut command = Command::new(image_capture.path());
9179    #[cfg(not(any(target_os = "macos", windows)))]
9180    let mut command = Command::new(&spec.program);
9181    command.args(&spec.args);
9182    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
9183    // that is the whole of the intent, so remove that one key rather than the
9184    // environment.
9185    //
9186    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
9187    // and took the POSIX environment with it. Modules spawned that way had no
9188    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
9189    // logging:
9190    //
9191    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
9192    //     both unset it fell back to the temp dir alone and `ck` could not find
9193    //     a daemon running on the same machine from inside any module's process
9194    //     tree — reporting a path the file has never lived at, which reads as
9195    //     "the daemon did not write its file".
9196    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
9197    //     the RELATIVE `.local/share`, so a module deriving its own store path
9198    //     resolved it against its own CWD. That is the store-fragmentation
9199    //     defect the daemon already refuses in config (`parse_doc` rejects a
9200    //     relative `storage.data_home`) arriving by derivation instead.
9201    //   * anything a module spawns inherited it: git without ~/.gitconfig,
9202    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
9203    //     quietly rather than erroring.
9204    //
9205    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
9206    // offered one candidate under /tmp while the file sat in /run/user/1000.
9207    //
9208    // A configured module is unaffected either way: `module_spec()` puts the
9209    // resolved CK_LOG into `spec.env`, which is applied below and therefore
9210    // wins over anything ambient.
9211    apply_child_env(&mut command, spec);
9212    apply_spawn_role(&mut command, role);
9213    let nonce_handoff =
9214        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9215
9216    #[cfg(target_os = "linux")]
9217    let cgroup_path = cgroup_placement
9218        .map(|placement| {
9219            if let Some(container) = &carried {
9220                let path = container
9221                    .cgroup_path()
9222                    .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9223                placement.existing_module_path(path)?.ok_or_else(|| {
9224                    io::Error::new(
9225                        io::ErrorKind::NotFound,
9226                        "carried cgroup disappeared before replacement",
9227                    )
9228                })
9229            } else {
9230                if carrying {
9231                    containment::prepare_cgroup_path(placement, &cgroup_name)
9232                } else {
9233                    placement.module_path(&cgroup_name)
9234                }
9235            }
9236        })
9237        .transpose()
9238        .map_err(|source| SuperviseError::Cgroup {
9239            module_id: spec.module_id.clone(),
9240            source,
9241        })?;
9242    #[cfg(not(target_os = "linux"))]
9243    let cgroup_path: Option<PathBuf> = None;
9244    #[cfg(unix)]
9245    let mut admission_gate = if carrying {
9246        Some(
9247            admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9248                SuperviseError::Spawn {
9249                    program: spec.program.clone(),
9250                    source,
9251                    cgroup_path: cgroup_path.clone(),
9252                }
9253            })?,
9254        )
9255    } else {
9256        None
9257    };
9258    #[cfg(target_os = "macos")]
9259    exec_ack.install(command.as_std_mut());
9260    #[cfg(target_os = "linux")]
9261    if let Some(path) = &cgroup_path {
9262        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9263            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9264                remove_module_cgroup(placement, &cgroup_name);
9265            }
9266            return Err(error);
9267        }
9268    }
9269
9270    let output_sink = if let Some(logs_dir) = capture_logs_dir {
9271        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9272        match ChildOutputSink::open(&path, capture_retention(spec)) {
9273            Ok(sink) => sink,
9274            Err(error) => {
9275                warn!(
9276                    module_id = %spec.module_id,
9277                    path = %path.display(),
9278                    error = %error,
9279                    "could not open child output capture file; forwarding to stderr"
9280                );
9281                ChildOutputSink::Stderr
9282            }
9283        }
9284    } else {
9285        ChildOutputSink::Stderr
9286    };
9287
9288    command.stdout(Stdio::piped());
9289    command.stderr(Stdio::piped());
9290    command.kill_on_drop(!carrying);
9291    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
9292    // before exec). In the daemon's group, a service manager that kills the
9293    // job's process group when the daemon exits (launchd's default) killed
9294    // every module at the same moment its control connection closed, so no
9295    // module ever ran its EOF teardown on a daemon stop. Outside that group a
9296    // module is reached only by the daemon: the EOF it sees when its
9297    // connection closes, and the bounded stop in `child_roster` for anything
9298    // still running after that. On Linux this composes with the cgroup
9299    // placement above: that is a pre_exec write to cgroup.procs, std performs
9300    // setpgid in the child before running pre_exec callbacks, and the two
9301    // change independent process attributes.
9302    //
9303    // stdin is /dev/null because a process outside the terminal's foreground
9304    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
9305    // by hand would otherwise hand down. Under a service manager stdin is
9306    // already /dev/null.
9307    #[cfg(unix)]
9308    command.process_group(
9309        carried
9310            .as_ref()
9311            .map(|container| container.verified_join_group())
9312            .transpose()
9313            .map_err(|source| SuperviseError::Spawn {
9314                program: spec.program.clone(),
9315                source,
9316                cgroup_path: cgroup_path.clone(),
9317            })?
9318            .flatten()
9319            .unwrap_or(0) as i32,
9320    );
9321    command.stdin(Stdio::null());
9322    #[cfg(unix)]
9323    if let Some(gate) = &mut admission_gate {
9324        gate.install_placement(&mut command);
9325    }
9326    // The LAST pre-exec step, after the cgroup placement above: installing the
9327    // nonce at descriptor 3 replaces whatever the child had there, which could
9328    // be the descriptor an earlier step writes through.
9329    #[cfg(unix)]
9330    if let Some(handoff) = nonce_handoff {
9331        handoff.install_last(command.as_std_mut());
9332    }
9333    #[cfg(not(any(unix, windows)))]
9334    let _ = nonce_handoff;
9335
9336    // Containment, step 1 of 3 (issue #109): create the child suspended so it
9337    // cannot run a single instruction -- and therefore cannot spawn a
9338    // grandchild -- before it is in the job. See `contain_spawned_child` for the
9339    // other two steps and why the window matters.
9340    #[cfg(windows)]
9341    {
9342        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9343            program: spec.program.clone(),
9344            source,
9345            cgroup_path: None,
9346        })?;
9347        subc_jobobject::suspend_on_create_async(&mut command);
9348    }
9349    // Timestamp before the spawn call: on Unix the child can serve before that
9350    // call even returns. This conservative bound never labels an older child
9351    // as a replacement for a request received during process creation.
9352    let spawned_at = Instant::now();
9353    #[cfg(all(test, unix))]
9354    containment_tests::record_spawn_attempt(spec, &mut command);
9355    #[cfg(unix)]
9356    let spawned = match admission_gate {
9357        Some(gate) => gate
9358            .spawn(
9359                command,
9360                spec,
9361                roster,
9362                launch_identity,
9363                #[cfg(target_os = "linux")]
9364                cgroup_placement
9365                    .zip(cgroup_path.as_ref())
9366                    .map(|(placement, path)| {
9367                        (placement.clone(), cgroup_name.clone(), path.clone())
9368                    }),
9369            )
9370            .map(|(child, admitted)| (child, Some(admitted))),
9371        None => command.spawn().map(|child| (child, None)),
9372    };
9373    #[cfg(not(unix))]
9374    let spawned = command.spawn();
9375    let spawned = match spawned {
9376        Ok(child) => child,
9377        Err(source) => {
9378            if carried
9379                .as_ref()
9380                .and_then(|container| container.group())
9381                .is_some()
9382            {
9383                containment::terminate_blocking(&roster.containment);
9384            }
9385            #[cfg(target_os = "linux")]
9386            if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9387                remove_module_cgroup(placement, &cgroup_name);
9388            }
9389            return Err(SuperviseError::Spawn {
9390                program: spec.program.clone(),
9391                source,
9392                cgroup_path,
9393            });
9394        }
9395    };
9396    #[cfg(unix)]
9397    let (mut child, mut admitted) = spawned;
9398    #[cfg(unix)]
9399    let group_target = admitted
9400        .as_ref()
9401        .map(|attempt| attempt.group_target())
9402        .unwrap_or(group_target);
9403    #[cfg(not(unix))]
9404    let mut child = spawned;
9405    #[cfg(test)]
9406    spawn_record_tests::after_create(&child, spec);
9407    // The parent must close its writer now: the acknowledgement pipe reports EOF
9408    // only when every writer is gone, and the child's copy closes when the
9409    // trampoline replaces itself with the module. Command holds only an integer
9410    // in its pre_exec callback, not another writer.
9411    #[cfg(target_os = "macos")]
9412    drop(exec_ack);
9413
9414    // Bind while the child is still suspended, before job assignment resumes it.
9415    // Keeping the executable handle open across CreateProcessW denies rename
9416    // and deletion, so the file cannot be exchanged during spawn. Bind that
9417    // file ID to the process handle returned by spawn, never a reopened PID.
9418    #[cfg(windows)]
9419    let windows_image = match image_capture.bind_tokio(&child) {
9420        Ok(image) => Arc::new(image),
9421        Err(source) => {
9422            kill_and_reap_rejected_child(&mut child);
9423            return Err(SuperviseError::Spawn {
9424                program: spec.program.clone(),
9425                source,
9426                cgroup_path: None,
9427            });
9428        }
9429    };
9430    // Assign the suspended child to its module job; resume only after its
9431    // process identity has been durably recorded for orphan recovery.
9432    #[cfg(windows)]
9433    let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9434    let spawned_at_ms = unix_ms_now();
9435    #[cfg(not(windows))]
9436    let spawned_from = spec.program.clone();
9437    #[cfg(windows)]
9438    let spawned_from = windows_image.path().to_path_buf();
9439    #[cfg(not(windows))]
9440    let spawned_file_identity = spawned_file_identity(&spawned_from);
9441    #[cfg(windows)]
9442    let spawned_file_identity = Some(SpawnedFileIdentity {
9443        device: 0,
9444        inode: 0,
9445        windows_image: Some(Arc::clone(&windows_image)),
9446    });
9447    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9448        program: spec.program.clone(),
9449        source: io::Error::other("spawned child exposed no live pid"),
9450        cgroup_path: cgroup_path.clone(),
9451    })?;
9452    #[cfg(windows)]
9453    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9454    #[cfg(not(windows))]
9455    let process_start_time = admitted
9456        .as_ref()
9457        .map(|admitted| admitted.birth)
9458        .or_else(|| crate::provenance::process_start_time(pid));
9459    #[cfg(windows)]
9460    let process_start_time = Some(windows_image.start_time());
9461    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9462    #[cfg(all(test, target_os = "macos"))]
9463    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9464    // Record the committed kernel image, not the configured pathname: PATH
9465    // lookup and shebang interpretation can select a different executable.
9466    // Linux's posix_spawn can return before that image is installed, so the
9467    // observer also waits for the kernel's first-exec flag to clear.
9468    #[cfg(not(windows))]
9469    let recorded_image = observe_spawned_image(pid);
9470    // spawn() confirms only the first exec, into the trampoline. Never persist
9471    // the trampoline image; the asynchronous acknowledgement publishes the
9472    // module image once the trampoline has replaced itself with the module.
9473    #[cfg(target_os = "macos")]
9474    let recorded_image = if privacy_exec.is_some() {
9475        None
9476    } else {
9477        recorded_image
9478    };
9479    #[cfg(target_os = "linux")]
9480    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9481    #[cfg(not(target_os = "linux"))]
9482    let recorded_cgroup_name = None;
9483    #[cfg(unix)]
9484    let preadmitted = admitted
9485        .as_mut()
9486        .and_then(|admitted| admitted.leader.take());
9487    #[cfg(not(unix))]
9488    let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9489    let roster_guard = if let Some(guard) = preadmitted {
9490        Ok(guard)
9491    } else {
9492        roster.admit(
9493            spec.module_id.clone(),
9494            pid,
9495            spec.protocol,
9496            process_start_time,
9497            crate::child_roster::RecordedIdentity {
9498                container: None,
9499                #[cfg(unix)]
9500                carried: None,
9501                #[cfg(not(windows))]
9502                start_time: recorded_image.map(|image| image.start_time),
9503                #[cfg(windows)]
9504                start_time: process_start_time,
9505                #[cfg(not(windows))]
9506                executable: recorded_image
9507                    .and_then(|image| image.executable)
9508                    .map(crate::live_children::ExecutableIdentity::from),
9509                #[cfg(windows)]
9510                executable: Some(crate::live_children::ExecutableIdentity::from(
9511                    windows_image.identity(),
9512                )),
9513                cgroup_name: recorded_cgroup_name,
9514                #[cfg(target_os = "linux")]
9515                cgroup_placement: cgroup_placement.cloned(),
9516            },
9517        )
9518    };
9519    let roster_guard = match roster_guard {
9520        Ok(guard) => guard,
9521        Err(failure) => {
9522            return failure.with_cleanup(|source| {
9523                #[cfg(windows)]
9524                if let Some(job) = &job {
9525                    let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9526                }
9527                #[cfg(target_os = "linux")]
9528                kill_module_cgroup(cgroup_placement, &cgroup_name);
9529                #[cfg(unix)]
9530                let rejected_group = group_target.clone();
9531                kill_and_reap_rejected_child(
9532                    &mut child,
9533                    #[cfg(unix)]
9534                    rejected_group,
9535                );
9536                #[cfg(target_os = "linux")]
9537                if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9538                    remove_module_cgroup(placement, &cgroup_name);
9539                }
9540                Err(SuperviseError::Spawn {
9541                    program: spec.program.clone(),
9542                    source: io::Error::other(format!(
9543                        "could not durably record module child: {source}"
9544                    )),
9545                    cgroup_path,
9546                })
9547            });
9548        }
9549    };
9550    #[cfg(unix)]
9551    if admitted.is_some() {
9552        if let Some(image) = recorded_image {
9553            if let Err(source) = roster_guard.confirm_executable(image) {
9554                kill_and_reap_rejected_child(&mut child, group_target.clone());
9555                return Err(SuperviseError::Spawn {
9556                    program: spec.program.clone(),
9557                    source,
9558                    cgroup_path,
9559                });
9560            }
9561        }
9562    }
9563    // The pre-spawn check can pass just before shutdown begins. Ownership is
9564    // published at the start of admission, even while a record write is queued.
9565    // If shutdown already saw an empty roster before that publication, this
9566    // check sees the roster closed and ends the child before starting a module.
9567    if roster.is_closed() {
9568        if carrying {
9569            #[cfg(unix)]
9570            let rejected_group = admitted
9571                .as_ref()
9572                .map(|admitted| admitted.rejection_target())
9573                .unwrap_or_else(|| group_target.clone());
9574            #[cfg(windows)]
9575            if let Some(job) = &job {
9576                let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9577            }
9578            kill_and_reap_rejected_child(
9579                &mut child,
9580                #[cfg(unix)]
9581                rejected_group,
9582            );
9583            #[cfg(all(test, unix))]
9584            let _ =
9585                roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9586            drop(roster_guard);
9587            return Err(SuperviseError::Spawn {
9588                program: spec.program.clone(),
9589                source: io::Error::other(
9590                    "the daemon began shutting down during carried admission; ended the container",
9591                ),
9592                cgroup_path,
9593            });
9594        }
9595        // This child was never admitted, so there is no module protocol shutdown to wait for.
9596        #[cfg(target_os = "linux")]
9597        kill_module_cgroup(cgroup_placement, &cgroup_name);
9598        if let Err(error) = child.start_kill() {
9599            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9600        }
9601        #[cfg(target_os = "linux")]
9602        if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9603            // This spawn was never admitted, so shutdown has no roster entry
9604            // to await. Do not detach its cleanup: the runtime could exit
9605            // before that task reaps the rejected child and removes its group.
9606            while matches!(child.try_wait(), Ok(None)) {
9607                std::thread::yield_now();
9608            }
9609            if matches!(
9610                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9611                subc_cgroup::KillOutcome::Killed
9612            ) {
9613                if let Ok(path) = placement.module_path(&cgroup_name) {
9614                    while std::fs::read_to_string(path.join("cgroup.events"))
9615                        .ok()
9616                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9617                    {
9618                        std::thread::yield_now();
9619                    }
9620                }
9621            }
9622            remove_module_cgroup(placement, &cgroup_name);
9623        }
9624        drop(roster_guard);
9625        return Err(SuperviseError::Spawn {
9626            program: spec.program.clone(),
9627            source: io::Error::other(
9628                "the daemon began shutting down while this process was starting; ended it",
9629            ),
9630            cgroup_path,
9631        });
9632    }
9633
9634    let stdout_pump = match child.stdout.take() {
9635        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9636        None => {
9637            warn!(
9638                module_id = %spec.module_id,
9639                "spawned child exposed no stdout pipe; file capture will be incomplete"
9640            );
9641            None
9642        }
9643    };
9644    let stderr_pump = match child.stderr.take() {
9645        Some(stderr) => {
9646            let generation = ring
9647                .lock()
9648                .unwrap_or_else(|poisoned| poisoned.into_inner())
9649                .begin_process();
9650            Some(StderrPump {
9651                task: tokio::spawn(pump_stderr_to(
9652                    stderr,
9653                    Arc::clone(ring),
9654                    generation,
9655                    output_sink,
9656                )),
9657                generation,
9658            })
9659        }
9660        None => {
9661            // Spawning succeeded but the pipe did not materialise. Recording it as
9662            // uncaptured keeps the tail honest: the alternative is an empty tail
9663            // that reads as a module which printed nothing.
9664            ring.lock()
9665                .unwrap_or_else(|poisoned| poisoned.into_inner())
9666                .mark_not_captured("stderr pipe was not available on spawn");
9667            warn!(
9668                module_id = %spec.module_id,
9669                "spawned child exposed no stderr pipe; tail will be unavailable"
9670            );
9671            None
9672        }
9673    };
9674
9675    let exec = Arc::new(crate::activity::ExecOutcome::new());
9676    #[cfg(target_os = "macos")]
9677    let pending_exec = privacy_exec.is_some();
9678    #[cfg(not(target_os = "macos"))]
9679    let pending_exec = false;
9680    if !pending_exec {
9681        exec.finish(crate::activity::ExecState::CommittedExec);
9682    }
9683    #[cfg(windows)]
9684    let subtree_target = job
9685        .as_ref()
9686        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9687        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9688    #[cfg(target_os = "linux")]
9689    let subtree_target = cgroup_path
9690        .map(crate::subtree_resources::Target::Cgroup)
9691        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9692    #[cfg(not(any(target_os = "linux", windows)))]
9693    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9694    let mut supervised = SupervisedChild {
9695        subtree_target,
9696        child: containment::ProcessChild::plain(child, pid),
9697        exec: crate::activity::ExecOwner(exec),
9698        #[cfg(windows)]
9699        nonce_delivery,
9700        protocol: spec.protocol,
9701        #[cfg(target_os = "linux")]
9702        module_id: cgroup_name,
9703        #[cfg(target_os = "linux")]
9704        cgroup_placement: cgroup_placement.cloned(),
9705        #[cfg(windows)]
9706        job,
9707        stdout_pump,
9708        stderr_pump,
9709        stderr_ring: Arc::clone(ring),
9710        spawned_at,
9711        registration_deadline: None,
9712        registration_watch: None,
9713        spawned_at_ms,
9714        spawned_from,
9715        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9716        spawned_file_identity,
9717        process_start_time,
9718        process_identity,
9719        pid,
9720        roster_guard: Some(roster_guard),
9721        #[cfg(target_os = "macos")]
9722        privacy_exec,
9723        #[cfg(target_os = "macos")]
9724        report_ready: Arc::new(OnceLock::new()),
9725        spawn_failure: None,
9726    };
9727    if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9728        supervised.child.reject(
9729            #[cfg(unix)]
9730            group_target,
9731        );
9732        #[cfg(all(test, unix))]
9733        let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9734        return Err(error);
9735    }
9736    #[cfg(unix)]
9737    if let Some(admitted) = admitted.take() {
9738        admitted.disarm();
9739    }
9740    #[cfg(windows)]
9741    if let Err(error) = resume_suspended_child(pid, spec) {
9742        containment::terminate_blocking(&roster.containment);
9743        let _ = supervised.child.start_kill();
9744        return Err(error);
9745    }
9746    Ok(supervised)
9747}
9748
9749#[cfg(test)]
9750mod spawn_record_tests {
9751    use super::*;
9752
9753    thread_local! {
9754        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9755    }
9756
9757    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9758        LAST_PID.set(child.id());
9759        #[cfg(windows)]
9760        if let Some((_, address)) = _spec
9761            .env
9762            .iter()
9763            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9764        {
9765            use std::io::{Read, Write};
9766            let mut stream = std::net::TcpStream::connect(address).unwrap();
9767            stream
9768                .write_all(&child.id().unwrap().to_ne_bytes())
9769                .unwrap();
9770            let mut release = [0];
9771            stream.read_exact(&mut release).unwrap();
9772        }
9773    }
9774
9775    #[cfg(windows)]
9776    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9777        if spec
9778            .env
9779            .iter()
9780            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9781        {
9782            Err(io::Error::other("injected module job assignment failure"))
9783        } else {
9784            Ok(())
9785        }
9786    }
9787
9788    fn spec() -> ModuleSpec {
9789        ModuleSpec {
9790            module_id: "spawn-record-worker".into(),
9791            #[cfg(unix)]
9792            program: "/bin/sleep".into(),
9793            #[cfg(windows)]
9794            program: std::env::current_exe().unwrap(),
9795            #[cfg(unix)]
9796            args: vec!["60".into()],
9797            #[cfg(windows)]
9798            args: vec![
9799                "--exact".into(),
9800                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9801                "--nocapture".into(),
9802            ],
9803            env: vec![],
9804            reserved: false,
9805            reserved_prefixes: vec![],
9806            protocol: ModuleProtocol::None,
9807            overlap: Default::default(),
9808            carry_containment: false,
9809        }
9810    }
9811
9812    #[tokio::test]
9813    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9814        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9815        let supervisor =
9816            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9817                .with_live_children_record(home.join("live-children.json"));
9818        crate::live_children::fail_next_file_sync();
9819        let result = supervisor.spawn(spec());
9820        let failed = result.is_err();
9821        if let Ok(module) = result {
9822            module.stop().await.unwrap();
9823        }
9824        assert!(failed, "record failure must fail spawn");
9825        let pid = LAST_PID.get().expect("the real OS child was created");
9826        assert!(
9827            subc_os::Process::open(pid).unwrap().is_none(),
9828            "child {pid} survived rejected admission or was not reaped"
9829        );
9830    }
9831
9832    #[cfg(target_os = "macos")]
9833    #[test]
9834    #[ignore = "manual on-device latency measurement"]
9835    fn durable_record_write_latency_50() {
9836        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9837        let path = home.join("live-children.json");
9838        let children = vec![crate::live_children::LiveChild {
9839            module_id: "latency-worker".into(),
9840            pid: std::process::id(),
9841            protocol: ModuleProtocol::None,
9842            start_time: Some(1),
9843            executable: None,
9844            cgroup_name: None,
9845            container: None,
9846        }];
9847        let mut samples = Vec::new();
9848        for _ in 0..50 {
9849            let start = std::time::Instant::now();
9850            crate::live_children::write_record(&path, &children).unwrap();
9851            samples.push(start.elapsed());
9852        }
9853        samples.sort();
9854        eprintln!(
9855            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9856            (samples[24] + samples[25]) / 2,
9857            samples[47]
9858        );
9859    }
9860
9861    #[cfg(windows)]
9862    #[test]
9863    fn windows_store_touch_fixture() {
9864        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9865            std::fs::write(path, b"module ran").unwrap();
9866        }
9867    }
9868
9869    #[cfg(windows)]
9870    #[tokio::test]
9871    async fn windows_daemon_fixture() {
9872        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9873            return;
9874        };
9875        let mut spec = spec();
9876        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9877        let supervisor =
9878            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9879        let _module = supervisor.spawn(spec).unwrap();
9880    }
9881
9882    #[cfg(windows)]
9883    #[test]
9884    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9885        use std::io::Read;
9886        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9887        // An outer job models a scheduler/service host. The daemon's root and
9888        // root job must nest without requiring breakaway from this job.
9889        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9890        command
9891            .args([
9892                "--exact",
9893                "supervise::spawn_record_tests::windows_daemon_fixture",
9894                "--nocapture",
9895            ])
9896            .env(
9897                "SUBC_TEST_DAEMON_BARRIER",
9898                listener.local_addr().unwrap().to_string(),
9899            );
9900        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9901        listener.set_nonblocking(true).unwrap();
9902        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9903        let mut stream = loop {
9904            if let Ok((stream, _)) = listener.accept() {
9905                break stream;
9906            }
9907            assert!(
9908                daemon.child.try_wait().unwrap().is_none(),
9909                "daemon failed before rendezvous"
9910            );
9911            assert!(
9912                std::time::Instant::now() < deadline,
9913                "daemon did not reach CreateProcess failpoint"
9914            );
9915            std::thread::yield_now();
9916        };
9917        stream
9918            .set_read_timeout(Some(Duration::from_secs(30)))
9919            .unwrap();
9920        let mut pid = [0; 4];
9921        stream.read_exact(&mut pid).unwrap();
9922        let pid = u32::from_ne_bytes(pid);
9923        assert!(subc_jobobject::process_exists(pid));
9924        // Terminate only the daemon, not its outer job: otherwise the host job
9925        // would also kill a child whose daemon never joined its root job.
9926        daemon.child.kill().unwrap();
9927        daemon.child.wait().unwrap();
9928        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9929        if !exited {
9930            daemon.job.terminate().unwrap();
9931        }
9932        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9933    }
9934
9935    #[cfg(windows)]
9936    #[tokio::test]
9937    async fn windows_containment_failure_never_resumes() {
9938        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9939        let sentinel = home.join("store-touched");
9940        let mut spec = spec();
9941        spec.env
9942            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9943        spec.env.push((
9944            "SUBC_TEST_STORE_TOUCH".into(),
9945            sentinel.display().to_string(),
9946        ));
9947        let supervisor =
9948            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9949        let result = supervisor.spawn(spec);
9950        // A version that resumes despite assignment failure must have time to
9951        // execute its sentinel before dropping the returned module kills it.
9952        if let Ok(module) = result {
9953            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9954                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9955                    tokio::task::yield_now().await;
9956                }
9957            })
9958            .await;
9959            let _ = module.stop().await;
9960            panic!(
9961                "assignment failure resumed a module; store touched={}",
9962                sentinel.exists()
9963            );
9964        }
9965        assert!(
9966            !sentinel.exists(),
9967            "refused module executed store-touch sentinel"
9968        );
9969        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9970    }
9971
9972    #[cfg(windows)]
9973    #[tokio::test]
9974    async fn windows_nested_daemon_fixture() {
9975        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9976            return;
9977        };
9978        let mut spec = spec();
9979        spec.env.push((
9980            "SUBC_TEST_STORE_TOUCH".into(),
9981            sentinel.to_str().unwrap().into(),
9982        ));
9983        let supervisor =
9984            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9985        let module = supervisor
9986            .spawn(spec)
9987            .expect("per-module job must nest below daemon and host jobs");
9988        tokio::time::timeout(Duration::from_secs(10), async {
9989            while !PathBuf::from(&sentinel).exists() {
9990                tokio::task::yield_now().await;
9991            }
9992        })
9993        .await
9994        .unwrap();
9995        module.stop().await.unwrap();
9996    }
9997
9998    #[cfg(windows)]
9999    #[test]
10000    fn windows_module_job_nests_under_host_job() {
10001        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
10002        let sentinel = home.join("nested-store-touched");
10003        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
10004        command
10005            .args([
10006                "--exact",
10007                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
10008                "--nocapture",
10009            ])
10010            .env("SUBC_TEST_NESTED_STORE", &sentinel);
10011        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
10012        assert!(daemon.child.wait().unwrap().success());
10013        assert!(sentinel.exists(), "nested protocol:none module never ran");
10014    }
10015}
10016
10017#[cfg(target_os = "linux")]
10018pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10019    use subc_cgroup::KillOutcome;
10020    match subc_cgroup::kill_module(placement, module_id) {
10021        KillOutcome::Killed => {}
10022        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10023            debug!(
10024                module_id,
10025                "cgroup tree kill unavailable; using direct-child kill"
10026            );
10027        }
10028        KillOutcome::IoError { path, error } => {
10029            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10030        }
10031    }
10032}
10033
10034/// Assign a freshly spawned Windows child without ever resuming on failure.
10035///
10036/// Steps 2 and 3 of the suspended-create contract: the job is created and the
10037/// child assigned **while it is still suspended** (step 1 is
10038/// `suspend_on_create_async` at the spawn site). The caller resumes only after
10039/// the child's roster admission is durable. Failed containment kills and reaps
10040/// the suspended child instead of running an unprotected module.
10041#[cfg(windows)]
10042fn contain_spawned_child(
10043    child: &mut Child,
10044    spec: &ModuleSpec,
10045    carried: Option<&Arc<containment::CarriedContainment>>,
10046) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10047    let assignment = || {
10048        let job = match carried {
10049            Some(container) => container.job(),
10050            None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10051        };
10052        #[cfg(test)]
10053        spawn_record_tests::assignment_failpoint(spec)?;
10054        job.lock()
10055            .unwrap_or_else(|p| p.into_inner())
10056            .assign(child)?;
10057        Ok::<_, io::Error>(job)
10058    };
10059    match assignment() {
10060        Ok(job) => Ok(Some(job)),
10061        Err(source) => {
10062            kill_and_reap_rejected_child(child);
10063            Err(SuperviseError::Spawn {
10064                program: spec.program.clone(),
10065                source: io::Error::other(format!(
10066                    "Windows module job containment failed: {source}"
10067                )),
10068                cgroup_path: None,
10069            })
10070        }
10071    }
10072}
10073
10074/// Rejection is synchronous: do not return a spawn error while the child can
10075/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
10076fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10077    #[cfg(unix)]
10078    group.signal_rejected(child);
10079    let _ = child.start_kill();
10080    while matches!(child.try_wait(), Ok(None)) {
10081        std::thread::yield_now();
10082    }
10083}
10084
10085/// Resume a suspended child, killing it if it cannot be started.
10086///
10087/// A suspended process holds a pid and does nothing, so there is no useful
10088/// state to return: the caller gets an error and the spawn fails.
10089#[cfg(windows)]
10090fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10091    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10092        // Kill it here rather than leaving a suspended process for the caller
10093        // to notice; `kill_on_drop` would eventually do this, but the module
10094        // would have been reported as running in between.
10095        let _ = std::process::Command::new("taskkill.exe")
10096            .args(["/PID", &pid.to_string(), "/T", "/F"])
10097            .stdin(Stdio::null())
10098            .stdout(Stdio::null())
10099            .stderr(Stdio::null())
10100            .status();
10101        return Err(SuperviseError::Spawn {
10102            program: spec.program.clone(),
10103            source,
10104            cgroup_path: None,
10105        });
10106    }
10107    Ok(())
10108}
10109
10110#[cfg(target_os = "linux")]
10111fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10112    match placement.remove_module(module_id) {
10113        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10114        Err(error) => warn!(
10115            module_id,
10116            error = %error,
10117            "could not remove module cgroup after process exit; continuing teardown"
10118        ),
10119    }
10120}
10121
10122#[cfg(target_os = "linux")]
10123const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10124#[cfg(target_os = "linux")]
10125const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10126
10127#[cfg(target_os = "linux")]
10128async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10129    let path = placement.module_path(module_id).ok();
10130    if path.as_ref().is_some_and(|path| {
10131        std::fs::read_to_string(path.join("cgroup.events"))
10132            .ok()
10133            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10134    }) {
10135        remove_module_cgroup(placement, module_id);
10136        return;
10137    }
10138
10139    // Freeze before enumerating: a frozen task cannot exit and release its PID
10140    // for reuse by a process outside this module while SIGTERM is being sent.
10141    if terminate_reaped_cgroup(placement, module_id).await {
10142        if let Some(path) = &path {
10143            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10144            while cgroup_is_populated(path) && Instant::now() < deadline {
10145                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10146            }
10147        }
10148    }
10149
10150    // Reaping the direct child is not proof its descendants exited. Kill any
10151    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
10152    // cgroup empty before rmdir.
10153    if matches!(
10154        subc_cgroup::kill_module(Some(placement), module_id),
10155        subc_cgroup::KillOutcome::Killed
10156    ) {
10157        if let Some(path) = &path {
10158            while cgroup_is_populated(path) {
10159                sleep(Duration::from_millis(1)).await;
10160            }
10161        }
10162    }
10163    remove_module_cgroup(placement, module_id);
10164}
10165
10166#[cfg(target_os = "linux")]
10167fn cgroup_is_populated(path: &std::path::Path) -> bool {
10168    std::fs::read_to_string(path.join("cgroup.events"))
10169        .ok()
10170        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10171}
10172
10173#[cfg(target_os = "linux")]
10174async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10175    let frozen =
10176        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10177            .await
10178        {
10179            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10180            outcome => {
10181                tracing::debug!(
10182                    module_id,
10183                    ?outcome,
10184                    "cgroup freeze failed; skipping SIGTERM grace"
10185                );
10186                return false;
10187            }
10188        };
10189    let listed = match frozen.processes() {
10190        subc_cgroup::ProcsOutcome::Listed(pids) => {
10191            for pid in pids.into_iter().filter(|pid| *pid > 1) {
10192                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10193                    if let Err(error) =
10194                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10195                    {
10196                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10197                    }
10198                }
10199            }
10200            true
10201        }
10202        outcome => {
10203            tracing::debug!(
10204                module_id,
10205                ?outcome,
10206                "cgroup process listing failed; skipping SIGTERM grace"
10207            );
10208            false
10209        }
10210    };
10211    match frozen.thaw() {
10212        subc_cgroup::ThawOutcome::Thawed => listed,
10213        outcome => {
10214            tracing::debug!(
10215                module_id,
10216                ?outcome,
10217                "cgroup thaw failed; proceeding to cgroup.kill"
10218            );
10219            false
10220        }
10221    }
10222}
10223
10224#[cfg(target_os = "linux")]
10225fn apply_cgroup_placement(
10226    command: &mut Command,
10227    spec: &ModuleSpec,
10228    path: &std::path::Path,
10229) -> Result<(), SuperviseError> {
10230    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10231        module_id: spec.module_id.clone(),
10232        source,
10233    })
10234}
10235
10236fn capture_retention(spec: &ModuleSpec) -> Retention {
10237    let defaults = Retention::default();
10238    let value = |name: &str| {
10239        spec.env
10240            .iter()
10241            .rev()
10242            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10243    };
10244    Retention {
10245        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10246            .and_then(|value| value.parse().ok())
10247            .unwrap_or(defaults.max_file_mb),
10248        keep: value(CAPTURE_KEEP_ENV)
10249            .and_then(|value| value.parse().ok())
10250            .unwrap_or(defaults.keep),
10251        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10252            .and_then(|value| value.parse().ok())
10253            .unwrap_or(defaults.max_age_days),
10254    }
10255}
10256
10257/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
10258/// module's registration to the exact process the supervisor spawned.
10259fn generate_launch_nonce() -> Result<String, SuperviseError> {
10260    let mut bytes = [0u8; 32];
10261    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10262        reason: source.to_string(),
10263    })?;
10264    let mut hex = String::with_capacity(64);
10265    for b in bytes {
10266        use std::fmt::Write;
10267        let _ = write!(hex, "{b:02x}");
10268    }
10269    Ok(hex)
10270}
10271
10272/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
10273/// signal about how many leading bytes matched.
10274fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10275    if a.len() != b.len() {
10276        return false;
10277    }
10278    let mut diff = 0u8;
10279    for (x, y) in a.iter().zip(b.iter()) {
10280        diff |= x ^ y;
10281    }
10282    diff == 0
10283}
10284
10285/// The kernel's image after an acknowledged exec, shared by ordinary launches
10286/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
10287/// not identities inferred from a configured pathname.
10288#[cfg(not(windows))]
10289fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10290    let process = subc_os::Process::open(pid).ok().flatten()?;
10291    #[cfg(target_os = "linux")]
10292    return sample_after_linux_exec(
10293        || {
10294            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10295            let committed = linux_exec_committed(&stat);
10296            #[cfg(test)]
10297            spawn_image_tests::report_exec_state(committed);
10298            committed
10299        },
10300        || process.observe(),
10301    );
10302    #[cfg(not(target_os = "linux"))]
10303    process.observe()
10304}
10305
10306/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
10307/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
10308/// installation. Until it clears, /proc/pid/exe can still name the daemon.
10309#[cfg(target_os = "linux")]
10310fn linux_exec_committed(stat: &str) -> Option<bool> {
10311    const PF_FORKNOEXEC: u64 = 0x40;
10312    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10313    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10314}
10315
10316#[cfg(target_os = "linux")]
10317fn sample_after_linux_exec(
10318    mut exec_committed: impl FnMut() -> Option<bool>,
10319    observe: impl FnOnce() -> Option<subc_os::Observation>,
10320) -> Option<subc_os::Observation> {
10321    let deadline = std::time::Instant::now() + Duration::from_secs(1);
10322    while !exec_committed()? {
10323        // A failed or indefinitely stalled exec must not publish the inherited
10324        // image as authority to signal a future orphan. Missing identity keeps
10325        // the existing conservative orphan-sweep behavior.
10326        if std::time::Instant::now() >= deadline {
10327            return None;
10328        }
10329        std::thread::sleep(Duration::from_micros(50));
10330    }
10331    observe()
10332}
10333
10334#[cfg(all(test, target_os = "linux"))]
10335mod spawn_image_tests {
10336    use super::*;
10337    use std::cell::{Cell, RefCell};
10338
10339    enum ExecEvent {
10340        Uncommitted,
10341        Image(Option<subc_os::Observation>),
10342    }
10343
10344    thread_local! {
10345        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10346    }
10347
10348    pub(super) fn report_exec_state(committed: Option<bool>) {
10349        if committed == Some(false) {
10350            EXEC_EVENTS.with(|events| {
10351                if let Some(events) = events.borrow_mut().take() {
10352                    let _ = events.send(ExecEvent::Uncommitted);
10353                }
10354            });
10355        }
10356    }
10357
10358    #[test]
10359    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10360        use std::io::{Read, Write};
10361        use std::os::fd::AsRawFd;
10362
10363        let descriptor = std::fs::File::open("/dev/null").unwrap();
10364        let mut command = std::process::Command::new("/bin/sleep");
10365        command.arg("30");
10366        let (mut ready, mut release) =
10367            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10368                .unwrap();
10369        let spawn = std::thread::spawn(move || command.spawn());
10370        let mut message = [0u8; 8];
10371        ready.read_exact(&mut message).unwrap();
10372        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10373        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10374        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10375        let (events, received) = std::sync::mpsc::channel();
10376        let observer = std::thread::spawn(move || {
10377            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10378            let image = observe_spawned_image(pid);
10379            let _ = events.send(ExecEvent::Image(image));
10380        });
10381        // Release only once the production observer has seen the uncommitted
10382        // exec, or has incorrectly returned the inherited image. Either way,
10383        // release and reap before asserting so a regression cannot leak a child.
10384        let first = received.recv_timeout(Duration::from_secs(5));
10385        release.write_all(&[1]).unwrap();
10386        let mut child = spawn.join().unwrap().unwrap();
10387        let after = match first {
10388            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10389                Ok(ExecEvent::Image(image)) => image,
10390                _ => None,
10391            },
10392            Ok(ExecEvent::Image(image)) => image,
10393            Err(_) => None,
10394        };
10395        observer.join().unwrap();
10396        child.kill().unwrap();
10397        child.wait().unwrap();
10398
10399        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10400        assert_eq!(
10401            inherited.unwrap().executable,
10402            subc_os::file_identity(&std::env::current_exe().unwrap())
10403        );
10404        assert_eq!(
10405            after.unwrap().executable,
10406            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10407        );
10408    }
10409
10410    #[test]
10411    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10412        let committed = Cell::new(false);
10413        let inherited = subc_os::Observation {
10414            start_time: 23,
10415            executable: Some(subc_os::FileIdentity {
10416                device: 29,
10417                inode: 31,
10418            }),
10419        };
10420        let module = subc_os::Observation {
10421            start_time: 23,
10422            executable: Some(subc_os::FileIdentity {
10423                device: 29,
10424                inode: 37,
10425            }),
10426        };
10427        let polls = Cell::new(0);
10428        let image = sample_after_linux_exec(
10429            || {
10430                polls.set(polls.get() + 1);
10431                // Hold the inherited image through the first poll, as in the
10432                // posix_spawn parent-wake window, then commit the module image.
10433                committed.set(polls.get() == 2);
10434                Some(committed.get())
10435            },
10436            || Some(if committed.get() { module } else { inherited }),
10437        );
10438        assert_eq!(image, Some(module));
10439        assert_eq!(polls.get(), 2);
10440    }
10441
10442    #[test]
10443    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10444        assert_eq!(
10445            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10446            Some(false)
10447        );
10448        assert_eq!(
10449            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10450            Some(true)
10451        );
10452        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10453        assert_eq!(linux_exec_committed("17 (a) R"), None);
10454    }
10455
10456    #[test]
10457    fn unreadable_exec_state_never_samples_an_inherited_image() {
10458        assert_eq!(
10459            sample_after_linux_exec(
10460                || None,
10461                || panic!("must not sample before exec confirmation")
10462            ),
10463            None
10464        );
10465    }
10466
10467    #[test]
10468    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10469        assert_eq!(
10470            sample_after_linux_exec(
10471                || Some(false),
10472                || panic!("must not sample after an exec timeout")
10473            ),
10474            None
10475        );
10476    }
10477}
10478
10479/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
10480/// different executable after the no-delete capture was taken.
10481#[cfg(windows)]
10482fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10483    let mut program = spec.program.clone();
10484    if program.is_absolute() || program.components().count() > 1 {
10485        // An explicit path can name a PE image with no extension. Try that
10486        // exact file first, as Command does, before falling back to .exe.
10487        match subc_os::ExecutableCapture::open(&program) {
10488            Ok(capture) => return Ok(capture),
10489            Err(error)
10490                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10491            Err(error) => return Err(error),
10492        }
10493        program.set_extension("exe");
10494        return subc_os::ExecutableCapture::open(&program);
10495    }
10496    if program.extension().is_none() {
10497        program.set_extension("exe");
10498    }
10499    let custom_path = spec
10500        .env
10501        .iter()
10502        .rev()
10503        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10504    let mut directories = Vec::new();
10505    // These are Windows' default executable search locations. A configured
10506    // PATH still overrides the inherited PATH, not the system directories.
10507    if let Some(parent) = std::env::current_exe()?.parent() {
10508        directories.push(parent.to_path_buf());
10509    }
10510    directories.push(std::env::current_dir()?);
10511    if let Some(windows) = std::env::var_os("SystemRoot") {
10512        let windows = PathBuf::from(windows);
10513        directories.push(windows.join("System32"));
10514        directories.push(windows.clone());
10515    }
10516    let path = custom_path
10517        .map(|(_, value)| std::ffi::OsString::from(value))
10518        .or_else(|| std::env::var_os("PATH"));
10519    if let Some(path) = path {
10520        directories.extend(std::env::split_paths(&path));
10521    }
10522    for directory in directories {
10523        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10524            Ok(capture) => return Ok(capture),
10525            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10526            Err(error) => return Err(error),
10527        }
10528    }
10529    Err(io::Error::new(
10530        io::ErrorKind::NotFound,
10531        "resolved executable not found",
10532    ))
10533}
10534
10535fn spawn_and_mark_running(
10536    spec: &ModuleSpec,
10537    runtime: &SupervisorRuntimeConfig,
10538    snapshot: &SharedSnapshot,
10539) -> Result<SupervisedChild, SuperviseError> {
10540    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10541    let registration_watch = watch_registration(&runtime.registry, spec);
10542    #[cfg(unix)]
10543    let joining_group = containment::current(&runtime.child_roster.containment)
10544        .is_some_and(|container| container.group().is_some());
10545    let spawn = || {
10546        spawn_child(
10547            spec,
10548            runtime.connection_file_path.as_deref(),
10549            runtime.supervisor_handle.as_ref(),
10550            &runtime.stderr_ring,
10551            runtime.capture_logs_dir.as_deref(),
10552            &runtime.child_roster,
10553            #[cfg(target_os = "linux")]
10554            runtime.cgroup_placement.as_ref(),
10555        )
10556    };
10557    let result = spawn();
10558    #[cfg(unix)]
10559    let result = if result.is_err()
10560        && joining_group
10561        && !runtime.child_roster.is_closed()
10562        && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10563        && runtime
10564            .child_roster
10565            .previous_instance(&spec.module_id)
10566            .is_none()
10567    {
10568        // The failed join already destroyed and verified the old container.
10569        // Try one fresh group as part of this same restart, without spending
10570        // another crash-budget entry. Cgroups and Windows jobs do not retry.
10571        warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10572        spawn()
10573    } else {
10574        result
10575    };
10576    let child = result?;
10577    let mut child = child;
10578    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10579    if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10580        containment::terminate_blocking(&runtime.child_roster.containment);
10581        return Err(error);
10582    }
10583    if let Some(spawn) = activity_spawn {
10584        spawn.complete(snapshot);
10585    }
10586    Ok(child)
10587}
10588
10589struct RegistrationWatch {
10590    changes: watch::Receiver<u64>,
10591    baseline: u64,
10592}
10593
10594// Capture before spawning: a fast child's accepted HELLO can disappear from the
10595// active registry before the monitor first runs. A pre-existing registration
10596// must not satisfy a new process's deadline either.
10597fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10598    if spec.protocol != ModuleProtocol::Subc {
10599        return None;
10600    }
10601    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10602    let baseline = *changes.borrow();
10603    Some(RegistrationWatch { changes, baseline })
10604}
10605
10606fn arm_registration_deadline(
10607    child: &mut SupervisedChild,
10608    budget: Duration,
10609    registration_watch: Option<RegistrationWatch>,
10610) {
10611    child.registration_deadline =
10612        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10613    child.registration_watch = registration_watch;
10614}
10615
10616async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10617    if let Some(container) = child.child.container() {
10618        container.stop_members().await;
10619        child.kill_leader()?;
10620        child.wait().await?;
10621        return Ok(());
10622    }
10623    #[cfg(unix)]
10624    {
10625        // Children lead their own process groups. Signal the whole group so a
10626        // helper stuck before HELLO cannot survive its supervised parent.
10627        let group = unreaped_registration_group(child)?;
10628        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10629        // Process-group safety rule: only signal a group while its leader is
10630        // our unreaped child. Even a zombie leader reserves its pid/group id;
10631        // reaping it before KILL would let that id name an unrelated group.
10632        // Sleep without waiting so TERM-ignoring helpers get the full grace.
10633        sleep(crate::child_roster::TERM_TO_KILL).await;
10634        let group = unreaped_registration_group(child)?;
10635        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10636    }
10637    // Windows terminates the containing Job Object as well as the direct
10638    // child. Linux also kills the child's cgroup when placement is enabled.
10639    child.start_kill()?;
10640    child.wait().await?;
10641    Ok(())
10642}
10643
10644#[cfg(unix)]
10645fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10646    if let Some(container) = child.child.container() {
10647        return container
10648            .group()
10649            .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10650            .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10651    }
10652    // The group id comes only from the pid recorded at spawn, when the child
10653    // created its own group. Never admit init's group or a negative group id.
10654    let group = i32::try_from(child.pid)
10655        .ok()
10656        .filter(|pid| *pid > 1)
10657        .and_then(rustix::process::Pid::from_raw)
10658        .ok_or_else(|| {
10659            io::Error::new(
10660                io::ErrorKind::InvalidInput,
10661                "invalid supervised process group",
10662            )
10663        })?;
10664    if child.child.id() != Some(child.pid) {
10665        return Err(io::Error::new(
10666            io::ErrorKind::InvalidInput,
10667            "refusing process-group signal after supervised child was reaped",
10668        ));
10669    }
10670    Ok(group)
10671}
10672
10673enum RegistrationWaitOutcome {
10674    Registered,
10675    Exited(ExitReport),
10676    TimedOut,
10677}
10678
10679struct ReloadRegistrationFailure {
10680    exit_report: ExitReport,
10681    reason: String,
10682}
10683
10684#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10685enum BusyGaugeObservation {
10686    Quiescent,
10687    Busy,
10688    Omitted,
10689}
10690
10691/// What one health answer said about one declared busy gauge.
10692#[derive(Debug, Clone, PartialEq, Eq)]
10693enum GaugeVerdict {
10694    /// The gauge read 0: this gauge is not what holds the drain.
10695    Zero,
10696    /// The gauge read a positive count.
10697    NonZero(u64),
10698    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10699    /// a negative number). It counts as busy, because the daemon cannot tell
10700    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10701    NotACount(String),
10702    /// The health answer had no metrics object, or one without this gauge.
10703    Missing,
10704    /// The probe produced no health answer to read; carries the probe error's
10705    /// evidence label (`no-answer` for a probe that timed out).
10706    ProbeFailed(&'static str),
10707}
10708
10709impl GaugeVerdict {
10710    fn reads_quiet(&self) -> bool {
10711        matches!(self, Self::Zero)
10712    }
10713}
10714
10715impl fmt::Display for GaugeVerdict {
10716    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10717        match self {
10718            Self::Zero => f.write_str("zero"),
10719            Self::NonZero(value) => write!(f, "non_zero({value})"),
10720            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10721            Self::Missing => f.write_str("missing_from_health_body"),
10722            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10723        }
10724    }
10725}
10726
10727/// The longest JSON text kept for a gauge value that is not a count, so a
10728/// module reporting a large object as a gauge cannot flood the drain log.
10729const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10730
10731/// Read each declared gauge out of one health answer's metrics.
10732fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10733    let metrics = metrics.and_then(Value::as_object);
10734    gauges
10735        .iter()
10736        .map(
10737            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10738                None => GaugeVerdict::Missing,
10739                Some(value) => match value.as_u64() {
10740                    Some(0) => GaugeVerdict::Zero,
10741                    Some(count) => GaugeVerdict::NonZero(count),
10742                    None => {
10743                        let mut text = value.to_string();
10744                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10745                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10746                            while !text.is_char_boundary(end) {
10747                                end -= 1;
10748                            }
10749                            text.truncate(end);
10750                            text.push_str("...");
10751                        }
10752                        GaugeVerdict::NotACount(text)
10753                    }
10754                },
10755            },
10756        )
10757        .collect()
10758}
10759
10760/// The drain's overall reading of one health answer: omitted when any
10761/// declared gauge is missing, quiescent only when every gauge reads 0.
10762fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10763    if verdicts
10764        .iter()
10765        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10766    {
10767        BusyGaugeObservation::Omitted
10768    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10769        BusyGaugeObservation::Quiescent
10770    } else {
10771        BusyGaugeObservation::Busy
10772    }
10773}
10774
10775/// One declared gauge's history over a drain: the latest verdict on it, and
10776/// the latest value actually read for it (which a later failed probe or
10777/// missing gauge does not erase). Times are offsets from the start of the
10778/// drain's wait.
10779#[derive(Debug, Clone, Default, PartialEq, Eq)]
10780struct GaugeTrace {
10781    latest: Option<(GaugeVerdict, Duration)>,
10782    last_value: Option<(GaugeVerdict, Duration)>,
10783}
10784
10785/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10786/// its ceiling can say in its timeout line which declared gauge held it and
10787/// why. Without it, a gauge that read busy, a gauge the module never
10788/// reported, and a probe that never got an answer all looked the same: a
10789/// drain that ended at its ceiling with nothing in flight.
10790#[derive(Debug, Clone, Default, PartialEq, Eq)]
10791struct DrainGaugeTrace {
10792    /// Health probes the drain sent.
10793    probes: u32,
10794    /// Each declared gauge, in declaration order.
10795    gauges: Vec<(String, GaugeTrace)>,
10796}
10797
10798impl DrainGaugeTrace {
10799    fn new(gauges: &[String]) -> Self {
10800        Self {
10801            probes: 0,
10802            gauges: gauges
10803                .iter()
10804                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10805                .collect(),
10806        }
10807    }
10808
10809    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10810    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10811        self.probes = self.probes.saturating_add(1);
10812        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10813            trace.latest = Some((verdict.clone(), at));
10814            if matches!(
10815                verdict,
10816                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10817            ) {
10818                trace.last_value = Some((verdict.clone(), at));
10819            }
10820        }
10821    }
10822}
10823
10824/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10825/// declared gauge, separated by ` | `. A value is shown as its verdict
10826/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10827/// and whether that counted as busy.
10828impl fmt::Display for DrainGaugeTrace {
10829    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10830        if self.gauges.is_empty() {
10831            return f.write_str("none declared");
10832        }
10833        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10834            if index > 0 {
10835                f.write_str(" | ")?;
10836            }
10837            write!(f, "{name}: ")?;
10838            match &trace.latest {
10839                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10840                None => f.write_str("never probed")?,
10841            }
10842            match &trace.last_value {
10843                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10844                None => f.write_str("; no value ever read")?,
10845            }
10846        }
10847        Ok(())
10848    }
10849}
10850
10851pub(crate) fn declared_busy_gauges(
10852    registry: &Registry,
10853    module_id: &str,
10854) -> Result<Vec<String>, SuperviseError> {
10855    busy_gauges_of(
10856        registry
10857            .get_module(module_id)
10858            .map_err(SuperviseError::Registry)?,
10859    )
10860}
10861
10862/// [`declared_busy_gauges`] for the registration a connection holds, in any
10863/// slot: after cutover the incumbent is no longer the id's active
10864/// registration, and its own manifest is the one that names its gauges.
10865fn declared_busy_gauges_for_connection(
10866    registry: &Registry,
10867    connection_id: ConnectionId,
10868) -> Result<Vec<String>, SuperviseError> {
10869    busy_gauges_of(
10870        registry
10871            .get_module_by_connection(connection_id)
10872            .map_err(SuperviseError::Registry)?,
10873    )
10874}
10875
10876fn busy_gauges_of(
10877    registration: Option<crate::registry::ModuleRegistration>,
10878) -> Result<Vec<String>, SuperviseError> {
10879    let Some(registration) = registration else {
10880        return Ok(Vec::new());
10881    };
10882    let Some(self_signals) = registration.manifest.self_signals else {
10883        return Ok(Vec::new());
10884    };
10885
10886    let mut gauges = Vec::new();
10887    for declaration in self_signals {
10888        if declaration.kind != SelfSignalKind::Busy {
10889            continue;
10890        }
10891        match declaration.anchored_to {
10892            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10893                gauges.extend(declared)
10894            }
10895            _ => {
10896                // An invalid Busy anchor is fail-safe: the empty name cannot be
10897                // present in a conforming health report, so this drain stays busy.
10898                gauges.push(String::new());
10899            }
10900        }
10901    }
10902    Ok(gauges)
10903}
10904
10905/// Wait for `endpoint` to have nothing in flight and, when the module declares
10906/// busy gauges, for a health probe to report them quiet. The probe is addressed
10907/// by `scope`: a swap's superseded incumbent must be asked about its own
10908/// gauges, and by module id the probe would reach the promoted candidate.
10909/// While the gauges read busy, the probe repeats every
10910/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10911/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10912/// timeout line.
10913#[allow(clippy::too_many_arguments)]
10914async fn wait_for_forwarding_quiescence(
10915    forwarding: &ForwardingTable,
10916    module_id: &str,
10917    runtime: &SupervisorRuntimeConfig,
10918    endpoint: crate::ModuleEndpointId,
10919    deadline: Instant,
10920    busy_gauges: &[String],
10921    scope: DrainScope,
10922    trace: &mut DrainGaugeTrace,
10923) -> Result<bool, SuperviseError> {
10924    let wait_started = Instant::now();
10925    let mut gauges_quiescent = busy_gauges.is_empty();
10926    let mut next_probe_at = wait_started;
10927    let mut omission_reported = false;
10928
10929    loop {
10930        let now = Instant::now();
10931        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10932            let report = match scope {
10933                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10934                DrainScope::Endpoint(endpoint) => {
10935                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10936                }
10937            };
10938            let verdicts = match &report {
10939                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10940                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10941            };
10942            trace.record(&verdicts, wait_started.elapsed());
10943            gauges_quiescent = match report {
10944                Ok(_) => match busy_gauge_observation(&verdicts) {
10945                    BusyGaugeObservation::Quiescent => true,
10946                    BusyGaugeObservation::Busy => false,
10947                    BusyGaugeObservation::Omitted => {
10948                        // Once per drain: a module that never reports a gauge
10949                        // it declared would otherwise say so on every probe.
10950                        if !omission_reported {
10951                            forwarding
10952                                .counters()
10953                                .increment_drains_with_undeclared_gauge();
10954                            let missing = busy_gauges
10955                                .iter()
10956                                .zip(&verdicts)
10957                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10958                                .map(|(gauge, _)| gauge.as_str())
10959                                .collect::<Vec<_>>()
10960                                .join(",");
10961                            warn!(
10962                                module_id,
10963                                missing_gauges = %missing,
10964                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10965                            );
10966                            omission_reported = true;
10967                        }
10968                        false
10969                    }
10970                },
10971                Err(err) => {
10972                    warn!(
10973                        module_id,
10974                        error = %err,
10975                        "drain health.check did not produce declared busy gauges; treating module as busy"
10976                    );
10977                    false
10978                }
10979            };
10980            // Counted from the probe's completion: the probe above was awaited,
10981            // so no second probe can start while one is still outstanding.
10982            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10983        }
10984
10985        let in_flight = forwarding
10986            .endpoint_in_flight_count(endpoint)
10987            .map_err(SuperviseError::Forwarding)?;
10988        if in_flight == 0 && gauges_quiescent {
10989            return Ok(true);
10990        }
10991
10992        let now = Instant::now();
10993        if now >= deadline {
10994            return Ok(false);
10995        }
10996        let mut wait = deadline
10997            .saturating_duration_since(now)
10998            .min(REGISTRY_RELEASE_POLL);
10999        if !busy_gauges.is_empty() {
11000            wait = wait.min(next_probe_at.saturating_duration_since(now));
11001        }
11002        sleep(wait).await;
11003    }
11004}
11005
11006/// The wait between drain busy-gauge probes: the larger of
11007/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
11008/// drain's own in-flight poll interval.
11009fn drain_busy_reprobe_interval() -> Duration {
11010    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
11011}
11012
11013/// Logs why a drain gave up: the drain waited its full timeout without the
11014/// module going quiet, and teardown is being forced.
11015///
11016/// The line names two kinds of evidence, because a bare "timed out" hides
11017/// causes that need different fixes:
11018/// - `holdouts`: the requests and routes still open when the timeout hit,
11019///   and on which connections. One stuck call is a module bug; a held stream
11020///   on every session is a module that should end its streams when it
11021///   receives `module.draining`.
11022/// - `gauges`: for each busy gauge the module declares, its latest reading
11023///   and how many probes ran. This separates a module that kept reporting
11024///   itself busy, one whose health answer never contained the gauge, and one
11025///   that never answered the probes, which otherwise all look like a timeout
11026///   with nothing in flight.
11027fn warn_drain_timed_out(
11028    module_id: &str,
11029    drain_timeout: Duration,
11030    reason: RouteCloseReason,
11031    holdouts: &crate::forwarding::DrainHoldouts,
11032    gauges: &DrainGaugeTrace,
11033) {
11034    warn!(
11035        module_id = %module_id,
11036        waited = ?drain_timeout,
11037        ?reason,
11038        held_requests = holdouts.requests,
11039        held_routes = holdouts.routes,
11040        total_routes = holdouts.total_routes,
11041        top_connections = ?holdouts.top_connections,
11042        // `module_channel:corr`, so the module can find each held request
11043        // in its own log; capped, so `held_requests` is the full count.
11044        held = %holdouts
11045            .held
11046            .iter()
11047            .map(|(channel, corr)| format!("{channel}:{corr}"))
11048            .collect::<Vec<_>>()
11049            .join(","),
11050        gauge_probes = gauges.probes,
11051        busy_gauges = %gauges,
11052        "route drain timed out before request quiescence; forcing teardown"
11053    );
11054}
11055
11056/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
11057///
11058/// `Ok` is always honest and passed straight through -- the wait actually measured
11059/// in-flight state. `Err` means the wait produced no measurement at all (the
11060/// forwarding table's lock was poisoned), so `false` is reported as the one honest
11061/// constant: the drain did not complete. Never recomputed from route state, never a
11062/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
11063fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11064    match wait_result {
11065        Ok(drained) => *drained,
11066        Err(_) => false,
11067    }
11068}
11069
11070fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11071    for released in released_routes {
11072        let frame = match Frame::build_with_version(
11073            released.negotiated_ver,
11074            FrameType::Goodbye,
11075            control_flags(),
11076            released.channel,
11077            released.epoch,
11078            0,
11079            Vec::new(),
11080        ) {
11081            Ok(frame) => frame,
11082            Err(err) => {
11083                warn!(
11084                    route_channel = released.channel,
11085                    error = %err,
11086                    "failed to build supervisor drain route GOODBYE frame"
11087                );
11088                continue;
11089            }
11090        };
11091        if !released.close_on_delivery_failure() {
11092            crate::forwarding::send_module_route_goodbye(
11093                &forwarding.counters(),
11094                &released.sink,
11095                frame,
11096                released.module_id.as_deref(),
11097                "supervisor drain",
11098            );
11099            continue;
11100        }
11101        if let Err(err) = released.sink.try_send(frame) {
11102            warn!(
11103                target_connection_id = released.connection_id.get(),
11104                route_channel = released.channel,
11105                error = %err,
11106                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11107            );
11108            let _ = forwarding.escalate_client_delivery_failure(
11109                released.connection_id,
11110                released.channel,
11111                released.epoch,
11112                CloseReason::new(
11113                    "route_goodbye_delivery_failed",
11114                    format!(
11115                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11116                        released.channel
11117                    ),
11118                ),
11119                crate::forwarding::UndeliveredFrame {
11120                    module_id: released.module_id.as_deref(),
11121                    sink: &released.sink,
11122                },
11123            );
11124        }
11125    }
11126}
11127
11128async fn send_module_draining(
11129    module_id: &str,
11130    reason: RouteCloseReason,
11131    deadline_ms: u64,
11132    target: &ModuleDrainTarget,
11133    _wait_for_flush: bool,
11134) -> bool {
11135    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11136        reason,
11137        deadline_ms,
11138    }) {
11139        Ok(body) => body,
11140        Err(err) => {
11141            warn!(
11142                module_id,
11143                error = %err,
11144                "failed to encode module draining command"
11145            );
11146            return false;
11147        }
11148    };
11149    let frame = match Frame::build_with_version(
11150        target.negotiated_ver,
11151        FrameType::Push,
11152        control_flags(),
11153        0,
11154        0,
11155        0,
11156        body,
11157    ) {
11158        Ok(frame) => frame,
11159        Err(err) => {
11160            warn!(
11161                module_id,
11162                error = %err,
11163                "failed to build module draining command frame"
11164            );
11165            return false;
11166        }
11167    };
11168    #[cfg(unix)]
11169    if _wait_for_flush {
11170        return target.sink.send_flushed(frame).await.is_ok();
11171    }
11172    if let Err(err) = target.sink.try_send(frame) {
11173        warn!(
11174            module_id,
11175            target_connection_id = target.endpoint.connection_id.get(),
11176            error = %err,
11177            "module draining command was not delivered to peer"
11178        );
11179        return false;
11180    }
11181    true
11182}
11183
11184/// The channel-0 GOODBYE that tells a module its stop is planned.
11185fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11186    match Frame::build_with_version(
11187        negotiated_ver,
11188        FrameType::Goodbye,
11189        control_flags(),
11190        0,
11191        0,
11192        0,
11193        Vec::new(),
11194    ) {
11195        Ok(frame) => Some(frame),
11196        Err(err) => {
11197            warn!(
11198                module_id,
11199                error = %err,
11200                "failed to build module GOODBYE frame"
11201            );
11202            None
11203        }
11204    }
11205}
11206
11207/// Send every registered module connection its module GOODBYE at daemon
11208/// shutdown, then request that connection's close.
11209///
11210/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
11211/// before EOF, so the GOODBYE must reach the socket before the close. A close
11212/// request does not wait for the connection's queued frames: its writer gets a
11213/// bounded grace after the close, is aborted if it overruns it, and the daemon
11214/// process may exit before that grace ends. So with `wait_for_flush`, each
11215/// connection is closed only after its writer has acknowledged writing the
11216/// GOODBYE, or once a short shared budget runs out, so one module that is not
11217/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
11218/// are only queued, for a shutdown the operator has told to stop waiting.
11219/// A connection that is already gone is skipped.
11220#[cfg(unix)]
11221async fn send_module_goodbyes_for_daemon_shutdown(
11222    forwarding: &Arc<ForwardingTable>,
11223    reason: &CloseReason,
11224    wait_for_flush: bool,
11225) {
11226    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11227    let targets = match forwarding.module_connections() {
11228        Ok(targets) => targets,
11229        Err(err) => {
11230            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11231            return;
11232        }
11233    };
11234    let deadline = Instant::now() + GOODBYE_BUDGET;
11235    let mut sends = tokio::task::JoinSet::new();
11236    for target in targets {
11237        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11238            continue;
11239        };
11240        if !wait_for_flush {
11241            if let Err(err) = target.sink.try_send(frame) {
11242                debug!(
11243                    module_id = %target.module_id,
11244                    error = %err,
11245                    "shutdown module GOODBYE was not queued"
11246                );
11247            }
11248            continue;
11249        }
11250        let forwarding = Arc::clone(forwarding);
11251        let reason = reason.clone();
11252        sends.spawn(async move {
11253            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11254                Ok(Ok(())) => {}
11255                Ok(Err(err)) => debug!(
11256                    module_id = %target.module_id,
11257                    error = %err,
11258                    "module connection closed before its shutdown GOODBYE was written"
11259                ),
11260                Err(_) => warn!(
11261                    module_id = %target.module_id,
11262                    budget = ?GOODBYE_BUDGET,
11263                    "shutdown module GOODBYE was not written within its budget; closing anyway"
11264                ),
11265            }
11266            forwarding.request_connection_close(target.endpoint.connection_id, reason);
11267        });
11268    }
11269    // Every task ends by the shared deadline, so this wait is bounded too.
11270    while sends.join_next().await.is_some() {}
11271}
11272
11273fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11274    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11275        return;
11276    };
11277    if let Err(err) = target.sink.try_send(frame) {
11278        warn!(
11279            module_id,
11280            target_connection_id = target.endpoint.connection_id.get(),
11281            error = %err,
11282            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11283        );
11284        forwarding.request_connection_close(
11285            target.endpoint.connection_id,
11286            CloseReason::new(
11287                "module_goodbye_delivery_failed",
11288                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11289            ),
11290        );
11291    }
11292}
11293
11294#[derive(Clone, Copy)]
11295struct ForwardingDrainContext<'a> {
11296    spec: &'a ModuleSpec,
11297    runtime: &'a SupervisorRuntimeConfig,
11298    registry: &'a Registry,
11299    scope: DrainScope,
11300}
11301
11302/// Which process a forwarding drain addresses.
11303#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11304enum DrainScope {
11305    /// Whatever endpoint is active for the module id: every plain stop,
11306    /// restart and reload. Also moves the module's state to `Draining`.
11307    Active,
11308    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
11309    /// module id would resolve to the promoted candidate and leave neither
11310    /// process routable. The module's state is left alone, since the promoted
11311    /// candidate is what it describes and that process is running.
11312    Endpoint(crate::ModuleEndpointId),
11313}
11314
11315/// Whether a child being drained has already been asked to stop by the time
11316/// its drain wait starts.
11317///
11318/// The drain wait is the same budget whatever this says. What it decides is
11319/// whether the supervisor must ask by signal before that wait begins: a child
11320/// that nobody asked will sit out the whole budget and then be SIGKILLed,
11321/// healthy or not.
11322#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11323enum StopNotice {
11324    /// The module was sent `module.draining` and a module GOODBYE over its own
11325    /// registered connection, and stops itself.
11326    SentOverConnection,
11327    /// The forwarding drain found no registered connection for the module: a
11328    /// subc child spawned moments ago that has not sent HELLO yet, or a
11329    /// `protocol: "none"` child, which never registers.
11330    NoConnection,
11331    /// This path sends nothing over the module's connection: the supervisor has
11332    /// no forwarding table, or the caller stops the child without a forwarding
11333    /// drain.
11334    NotSent,
11335}
11336
11337async fn begin_forwarding_drain(
11338    spec: &ModuleSpec,
11339    runtime: &SupervisorRuntimeConfig,
11340    registry: &Registry,
11341    snapshot: &SharedSnapshot,
11342    enabled: Option<bool>,
11343    reason: RouteCloseReason,
11344) -> Result<StopNotice, SuperviseError> {
11345    let Some(forwarding) = runtime.forwarding.as_ref() else {
11346        return Err(SuperviseError::ReloadUnavailable {
11347            module_id: spec.module_id.clone(),
11348            reason: "supervisor was not configured with a forwarding table".to_string(),
11349        });
11350    };
11351
11352    begin_forwarding_drain_with(
11353        forwarding,
11354        ForwardingDrainContext {
11355            spec,
11356            runtime,
11357            registry,
11358            scope: DrainScope::Active,
11359        },
11360        snapshot,
11361        enabled,
11362        reason,
11363        runtime.drain_timeout,
11364    )
11365    .await
11366}
11367
11368async fn begin_forwarding_drain_if_configured(
11369    spec: &ModuleSpec,
11370    runtime: &SupervisorRuntimeConfig,
11371    registry: &Registry,
11372    snapshot: &SharedSnapshot,
11373    enabled: Option<bool>,
11374    reason: RouteCloseReason,
11375) -> Result<StopNotice, SuperviseError> {
11376    begin_forwarding_drain_with_timeout(
11377        spec,
11378        runtime,
11379        registry,
11380        snapshot,
11381        enabled,
11382        reason,
11383        runtime.drain_timeout,
11384    )
11385    .await
11386}
11387
11388/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
11389/// budget, for paths where the operator overrides the module's configured one
11390/// (`supervisor.restart{drain_timeout_ms}`).
11391async fn begin_forwarding_drain_with_timeout(
11392    spec: &ModuleSpec,
11393    runtime: &SupervisorRuntimeConfig,
11394    registry: &Registry,
11395    snapshot: &SharedSnapshot,
11396    enabled: Option<bool>,
11397    reason: RouteCloseReason,
11398    drain_timeout: Duration,
11399) -> Result<StopNotice, SuperviseError> {
11400    let Some(forwarding) = runtime.forwarding.as_ref() else {
11401        return Ok(StopNotice::NotSent);
11402    };
11403
11404    begin_forwarding_drain_with(
11405        forwarding,
11406        ForwardingDrainContext {
11407            spec,
11408            runtime,
11409            registry,
11410            scope: DrainScope::Active,
11411        },
11412        snapshot,
11413        enabled,
11414        reason,
11415        drain_timeout,
11416    )
11417    .await
11418}
11419
11420async fn begin_forwarding_drain_with(
11421    forwarding: &ForwardingTable,
11422    context: ForwardingDrainContext<'_>,
11423    snapshot: &SharedSnapshot,
11424    enabled: Option<bool>,
11425    reason: RouteCloseReason,
11426    drain_timeout: Duration,
11427) -> Result<StopNotice, SuperviseError> {
11428    let ForwardingDrainContext {
11429        spec,
11430        runtime,
11431        registry,
11432        scope,
11433    } = context;
11434    debug_assert_ne!(reason, RouteCloseReason::Crash);
11435    let terminal = matches!(reason, RouteCloseReason::Disable);
11436    let drain_started_at = Instant::now();
11437    let drain_deadline = drain_started_at + drain_timeout;
11438    let deadline_ms =
11439        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11440    let busy_gauges = match scope {
11441        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11442        DrainScope::Endpoint(endpoint) => {
11443            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11444        }
11445    };
11446
11447    // Admission gate first: route.open/commit and route REQUEST admission are closed
11448    // before the first quiescence check, so the outstanding count can only fall.
11449    let gate_started = Instant::now();
11450    let drain_target = match scope {
11451        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11452        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11453    }
11454    .map_err(SuperviseError::Forwarding)?;
11455    // The instant admission closed, and how long taking the forwarding write
11456    // lock to close it took. The timeout line reports only the quiescence
11457    // wait, so without this a drain that started late looked like one that
11458    // started on time.
11459    info!(
11460        module_id = %spec.module_id,
11461        ?reason,
11462        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11463        connected = drain_target.is_some(),
11464        "module drain began; route admission closed"
11465    );
11466    if scope == DrainScope::Active {
11467        update_snapshot(snapshot, Some(&spec.module_id), |state| {
11468            state.state = ModuleState::Draining;
11469            state.draining_to_replace =
11470                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11471            if let Some(enabled) = enabled {
11472                state.enabled = enabled;
11473            }
11474        })?;
11475    }
11476
11477    let Some(target) = drain_target.as_ref() else {
11478        // Nothing was sent: the module has no registered connection to carry
11479        // `module.draining` or a GOODBYE. The caller must not assume the child
11480        // was asked to stop.
11481        return Ok(StopNotice::NoConnection);
11482    };
11483    {
11484        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11485        #[cfg(feature = "test-support")]
11486        forwarding.wait_before_route_closing_for_test().await;
11487        let routes = forwarding
11488            .endpoint_routes(target.endpoint)
11489            .map_err(SuperviseError::Forwarding)?;
11490        let routes_notified = routes.len();
11491        crate::control::send_route_control_pushes(
11492            forwarding,
11493            routes.clone(),
11494            ClientControlPush::RouteClosing {
11495                module_id: spec.module_id.clone(),
11496                channels: Vec::new(),
11497                reason,
11498            },
11499        );
11500        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11501
11502        // `route.closing` was just sent above: from here on every return path,
11503        // including an early one, MUST send `route.closed` before propagating
11504        // anything else. A client holds `closing` as a promise that a verdict is
11505        // coming; leaving early without `closed` strands it waiting forever, since
11506        // `closing` carries no timeout of its own.
11507        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11508        let wait_result = wait_for_forwarding_quiescence(
11509            forwarding,
11510            &spec.module_id,
11511            runtime,
11512            target.endpoint,
11513            drain_deadline,
11514            &busy_gauges,
11515            scope,
11516            &mut gauge_trace,
11517        )
11518        .await;
11519        let drained = drained_after_quiescence_wait(&wait_result);
11520        if let Err(err) = &wait_result {
11521            error!(
11522                module_id = %spec.module_id,
11523                ?reason,
11524                error = %err,
11525                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11526            );
11527        } else if !drained {
11528            // Read before teardown releases the routes.
11529            let holdouts = forwarding
11530                .endpoint_drain_holdouts(target.endpoint)
11531                .unwrap_or_default();
11532            warn_drain_timed_out(
11533                &spec.module_id,
11534                drain_timeout,
11535                reason,
11536                &holdouts,
11537                &gauge_trace,
11538            );
11539        }
11540        crate::control::send_route_control_pushes(
11541            forwarding,
11542            routes,
11543            ClientControlPush::RouteClosed {
11544                module_id: spec.module_id.clone(),
11545                channels: Vec::new(),
11546                reason,
11547                drained,
11548                abandoned: target.abandoned_bindings.len() as u32,
11549                excluded_subscriptions: target.excluded_subscriptions,
11550                terminal: Some(terminal),
11551            },
11552        );
11553        wait_result?;
11554
11555        // `route.closed` has now been sent unconditionally above. From here the
11556        // remaining steps are cleanup (route + module GOODBYE) rather than a
11557        // promise the client is waiting on, but a lock-poisoned
11558        // `release_module_endpoint_routes` would otherwise skip the module
11559        // GOODBYE silently too -- send it before propagating the error.
11560        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11561            Ok(routes) => routes,
11562            Err(err) => {
11563                warn!(
11564                    module_id = %spec.module_id,
11565                    ?reason,
11566                    error = %err,
11567                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11568                );
11569                send_module_goodbye(&spec.module_id, forwarding, target);
11570                return Err(SuperviseError::Forwarding(err));
11571            }
11572        };
11573        let route_goodbye_count = released_routes.len();
11574        send_route_goodbyes(forwarding, released_routes);
11575        send_module_goodbye(&spec.module_id, forwarding, target);
11576
11577        // The drain's happy path was previously silent: every emission above is
11578        // best-effort with only its failure arm logged, so "were consumers told"
11579        // was unprovable from the daemon log (surfaced by a 30-minute consumer
11580        // hang where the open question was exactly whether teardown notice went
11581        // out). One summary line makes that class decidable in one grep.
11582        info!(
11583            module_id = %spec.module_id,
11584            ?reason,
11585            routes_notified,
11586            route_goodbyes = route_goodbye_count,
11587            abandoned_reservations = target.abandoned_bindings.len(),
11588            excluded_subscriptions = target.excluded_subscriptions,
11589            drained,
11590            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11591        );
11592    }
11593
11594    Ok(StopNotice::SentOverConnection)
11595}
11596
11597/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
11598/// the only slot a plain (non-swap) spawn can register into.
11599async fn wait_for_registration_after_reload(
11600    registry: &Registry,
11601    module_id: &str,
11602    snapshot: &SharedSnapshot,
11603    child: &mut SupervisedChild,
11604    wait: Duration,
11605) -> Result<RegistrationWaitOutcome, SuperviseError> {
11606    wait_for_slot_registration(
11607        registry,
11608        crate::registry::RegistrationSlot::Active(module_id),
11609        module_id,
11610        snapshot,
11611        child,
11612        wait,
11613    )
11614    .await
11615}
11616
11617/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
11618///
11619/// Keyed on the slot rather than the bare module id because during a swap the
11620/// id's active slot is already held by the incumbent: an id-keyed wait would
11621/// report the incumbent's registration as the candidate's and a candidate that
11622/// never registers would look registered. A swap candidate waits on
11623/// `crate::registry::RegistrationSlot::Candidate`.
11624async fn wait_for_slot_registration(
11625    registry: &Registry,
11626    slot: crate::registry::RegistrationSlot<'_>,
11627    module_id: &str,
11628    snapshot: &SharedSnapshot,
11629    child: &mut SupervisedChild,
11630    wait: Duration,
11631) -> Result<RegistrationWaitOutcome, SuperviseError> {
11632    let deadline = Instant::now() + wait;
11633    loop {
11634        if registry
11635            .registration(slot)
11636            .map_err(SuperviseError::Registry)?
11637            .is_some()
11638        {
11639            return Ok(RegistrationWaitOutcome::Registered);
11640        }
11641
11642        let now = Instant::now();
11643        if now >= deadline {
11644            return Ok(RegistrationWaitOutcome::TimedOut);
11645        }
11646        let remaining = deadline.saturating_duration_since(now);
11647        let poll = remaining.min(REGISTRY_RELEASE_POLL);
11648
11649        tokio::select! {
11650            wait_result = child.wait() => {
11651                let status = wait_result.map_err(|source| SuperviseError::Wait {
11652                    module_id: module_id.to_string(),
11653                    source,
11654                })?;
11655                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11656                    snapshot,
11657                    child,
11658                    &status,
11659                )));
11660            }
11661            _ = sleep(poll) => {}
11662        }
11663    }
11664}
11665
11666fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11667    // A process stopped for missing its first HELLO did not provide service,
11668    // even if its TERM handler exits zero. Reload also uses this classification
11669    // when a replacement exits before registering.
11670    if exit_report.kind != ExitKind::DeliberateSeverance {
11671        exit_report.kind = ExitKind::Crash;
11672    }
11673    exit_report
11674}
11675
11676async fn handle_reload_child_registration_failure(
11677    spec: &ModuleSpec,
11678    runtime: &SupervisorRuntimeConfig,
11679    registry: &Registry,
11680    process_liveness: &SupervisorProcessLiveness,
11681    snapshot: &SharedSnapshot,
11682    _child: &mut Option<SupervisedChild>,
11683    failure: ReloadRegistrationFailure,
11684) -> Result<(), SuperviseError> {
11685    let ReloadRegistrationFailure {
11686        exit_report,
11687        reason,
11688    } = failure;
11689    match on_child_exit(
11690        spec,
11691        runtime.restart_policy,
11692        registry,
11693        snapshot,
11694        &runtime.terminal_ring,
11695        &runtime.spawn_events,
11696        &runtime.child_roster,
11697        exit_report,
11698        runtime.recovery_is_daemon,
11699    )
11700    .await
11701    {
11702        NextAction::Stop {
11703            registration_released,
11704        } => {
11705            if registration_released {
11706                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11707            }
11708        }
11709        NextAction::Restart { schedule } => {
11710            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11711                schedule.delay
11712            });
11713            if let Some(schedule) = schedule {
11714                log_crash_respawn(&spec.module_id, schedule);
11715            }
11716            schedule_respawn(
11717                runtime,
11718                snapshot,
11719                &spec.module_id,
11720                delay,
11721                RespawnKind::Spawn,
11722                true,
11723            )?;
11724        }
11725    }
11726    Err(SuperviseError::ReloadFailed {
11727        module_id: spec.module_id.clone(),
11728        reason,
11729    })
11730}
11731
11732async fn handle_reload_spawn_failure(
11733    spec: &ModuleSpec,
11734    runtime: &SupervisorRuntimeConfig,
11735    process_liveness: &SupervisorProcessLiveness,
11736    snapshot: &SharedSnapshot,
11737    _child: &mut Option<SupervisedChild>,
11738    reason: String,
11739) -> Result<(), SuperviseError> {
11740    let now = Instant::now();
11741    let mut schedule = None;
11742    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11743        clear_current_process_facts(state);
11744        if state.enabled {
11745            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11746            state.state = if schedule.is_some() {
11747                ModuleState::Restarting
11748            } else {
11749                if runtime
11750                    .failure_context(runtime.recovery_is_daemon)
11751                    .may_retry()
11752                {
11753                    state.failed_retry.schedule(&spec.module_id);
11754                }
11755                ModuleState::Failed
11756            };
11757        } else {
11758            state.state = ModuleState::Disabled;
11759        }
11760    })?;
11761    if let Some(schedule) = schedule {
11762        schedule_respawn(
11763            runtime,
11764            snapshot,
11765            &spec.module_id,
11766            schedule.delay,
11767            RespawnKind::Spawn,
11768            true,
11769        )?;
11770    } else {
11771        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11772    }
11773    Err(SuperviseError::ReloadFailed {
11774        module_id: spec.module_id.clone(),
11775        reason,
11776    })
11777}
11778
11779fn control_flags() -> Flags {
11780    Flags::new(false, Priority::Passive, false)
11781}
11782
11783#[allow(clippy::too_many_arguments)]
11784async fn drain_optional_child(
11785    module_id: &str,
11786    protocol: ModuleProtocol,
11787    stop_notice: StopNotice,
11788    registry: &Registry,
11789    forwarding: Option<&ForwardingTable>,
11790    snapshot: &SharedSnapshot,
11791    terminal_ring: &Arc<Mutex<TerminalRing>>,
11792    spawn_events: &SpawnEventFeed,
11793    child: &mut Option<SupervisedChild>,
11794    drain_timeout: Duration,
11795    final_state: ModuleState,
11796    enabled: Option<bool>,
11797    failure: &FailureContext,
11798) -> Result<(), SuperviseError> {
11799    if let Some(child) = child.take() {
11800        drain_child_to_state(
11801            module_id,
11802            protocol,
11803            stop_notice,
11804            registry,
11805            forwarding,
11806            snapshot,
11807            terminal_ring,
11808            spawn_events,
11809            child,
11810            drain_timeout,
11811            final_state,
11812            enabled,
11813            failure,
11814        )
11815        .await
11816    } else {
11817        update_snapshot(snapshot, Some(module_id), |state| {
11818            state.state = final_state;
11819            if final_state == ModuleState::Failed {
11820                state.failed_retry.failed_at = Some(Instant::now());
11821            }
11822            if let Some(enabled) = enabled {
11823                state.enabled = enabled;
11824            }
11825            clear_current_process_facts(state);
11826        })?;
11827        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11828    }
11829}
11830
11831#[allow(clippy::too_many_arguments)]
11832async fn drain_child_to_state(
11833    module_id: &str,
11834    _protocol: ModuleProtocol,
11835    stop_notice: StopNotice,
11836    registry: &Registry,
11837    forwarding: Option<&ForwardingTable>,
11838    snapshot: &SharedSnapshot,
11839    terminal_ring: &Arc<Mutex<TerminalRing>>,
11840    spawn_events: &SpawnEventFeed,
11841    mut child: SupervisedChild,
11842    drain_timeout: Duration,
11843    final_state: ModuleState,
11844    enabled: Option<bool>,
11845    failure: &FailureContext,
11846) -> Result<(), SuperviseError> {
11847    let protocol = child.protocol;
11848    update_snapshot(snapshot, Some(module_id), |state| {
11849        state.state = ModuleState::Draining;
11850        state.draining_to_replace = final_state == ModuleState::Restarting;
11851        if let Some(enabled) = enabled {
11852            state.enabled = enabled;
11853        }
11854    })?;
11855
11856    // The wait below is the same budget in every case; what differs is
11857    // whether anything has ASKED the child to stop before it starts. Only a
11858    // forwarding drain that reached the module's registered connection has
11859    // (`module.draining`, then a module GOODBYE). Every other child was told
11860    // nothing: a `protocol: "none"` module, which never registers; a subc
11861    // module spawned moments ago that has not sent HELLO yet; or a stop that
11862    // runs no forwarding drain. Without a signal the budget is only a delay
11863    // in front of SIGKILL -- and the not-yet-registered child is the worst
11864    // case, because it registers into a module that is already draining,
11865    // is never told, and is killed while healthy.
11866    if stop_notice != StopNotice::SentOverConnection {
11867        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11868            info!(
11869                module_id,
11870                pid = child.pid,
11871                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11872                "module has no connection yet; requesting stop by signal"
11873            );
11874        }
11875        request_graceful_stop(module_id, &child);
11876    }
11877
11878    let exit_report = match timeout(drain_timeout, child.wait()).await {
11879        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11880        Ok(Err(source)) => {
11881            fail_after_wait_error(
11882                module_id,
11883                snapshot,
11884                terminal_ring,
11885                spawn_events,
11886                format!("failed to wait during drain: {source}"),
11887                failure,
11888            );
11889            return Err(SuperviseError::Wait {
11890                module_id: module_id.to_string(),
11891                source,
11892            });
11893        }
11894        Err(_) => {
11895            // Mirror the sibling arm above: state is already `Draining`, and an
11896            // error propagated from here would strand it there -- a state
11897            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11898            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11899            // `Failed` before `?` keeps the module operator-visible and
11900            // revivable. Trigger is an ESRCH race (process exits between the
11901            // drain timeout firing and the kill) or a post-kill wait failure
11902            // (issue #34).
11903            //
11904            // Logged because the kill is otherwise visible only as signal 9 in
11905            // the terminal ring, and the budget it follows can be long enough
11906            // that consumers see a stretch of refusals with no stated cause.
11907            warn!(
11908                module_id,
11909                pid = child.pid,
11910                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11911                reason = ?final_state,
11912                ?stop_notice,
11913                "drain budget expired before the module exited; killing it"
11914            );
11915            let kill =
11916                if final_state == ModuleState::Restarting && child.child.container().is_some() {
11917                    child.kill_leader()
11918                } else {
11919                    child.start_kill()
11920                };
11921            kill.map_err(|source| {
11922                fail_after_wait_error(
11923                    module_id,
11924                    snapshot,
11925                    terminal_ring,
11926                    spawn_events,
11927                    format!("failed to kill during drain: {source}"),
11928                    failure,
11929                );
11930                SuperviseError::Kill {
11931                    module_id: module_id.to_string(),
11932                    source,
11933                }
11934            })?;
11935            let status = child.wait().await.map_err(|source| {
11936                fail_after_wait_error(
11937                    module_id,
11938                    snapshot,
11939                    terminal_ring,
11940                    spawn_events,
11941                    format!("failed to wait after drain kill: {source}"),
11942                    failure,
11943                );
11944                SuperviseError::Wait {
11945                    module_id: module_id.to_string(),
11946                    source,
11947                }
11948            })?;
11949            classify_reaped_child_exit(snapshot, &child, &status)
11950        }
11951    };
11952
11953    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11954    publish_after_terminal_record(
11955        module_id,
11956        || {
11957            record_terminal_with_detail(
11958                module_id,
11959                terminal_ring,
11960                spawn_events,
11961                &exit_report,
11962                terminal_disposition(final_state),
11963                detail,
11964            )
11965        },
11966        || {
11967            update_snapshot(snapshot, Some(module_id), |state| {
11968                state.state = final_state;
11969                if final_state == ModuleState::Failed {
11970                    state.failed_retry.failed_at = Some(Instant::now());
11971                }
11972                if let Some(enabled) = enabled {
11973                    state.enabled = enabled;
11974                }
11975                clear_current_process_facts(state);
11976                state.last_exit = Some(exit_report.clone());
11977                if exit_report.kind == ExitKind::DeliberateSeverance {
11978                    state.lifetime_restarts += 1;
11979                }
11980            })
11981        },
11982    )?;
11983    child.drain_stderr(module_id).await;
11984
11985    let released =
11986        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11987    child
11988        .finish_exit(
11989            released.is_ok()
11990                && final_state == ModuleState::Restarting
11991                && child.spawn_failure.is_none(),
11992        )
11993        .await
11994        .map_err(|source| SuperviseError::Wait {
11995            module_id: module_id.to_owned(),
11996            source,
11997        })?;
11998    released
11999}
12000
12001/// Ask a child that nothing else has asked to stop, by signal.
12002///
12003/// A registered subc module is asked over its own connection: the drain sends
12004/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
12005/// module GOODBYE, and the module stops itself. A module that speaks no subc
12006/// wire receives none of that, and neither does a subc module that has not
12007/// registered yet, so for them the drain budget would be pure delay in front of
12008/// a SIGKILL -- and for a process with a store to flush (JetStream is the
12009/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
12010/// into a recovery on the next start.
12011///
12012/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
12013/// rule rather than an optimisation: that module's graceful stop is already
12014/// running by the time its child is drained, and a signal would race it.
12015///
12016/// Best-effort by construction. A child that has already exited is the ordinary
12017/// case rather than an error (the kill lands on a reaped or exiting pid), so a
12018/// failure is logged at debug and the wait-then-kill below still decides the
12019/// outcome.
12020#[cfg(unix)]
12021fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12022    let Some(pid) = child
12023        .id()
12024        .and_then(|pid| i32::try_from(pid).ok())
12025        .and_then(rustix::process::Pid::from_raw)
12026    else {
12027        debug!(
12028            module_id,
12029            "no pid to signal for teardown; falling through to the drain wait"
12030        );
12031        return;
12032    };
12033    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12034        Ok(()) => debug!(
12035            module_id,
12036            "sent SIGTERM to a module nothing else asked to stop"
12037        ),
12038        Err(err) => debug!(
12039            module_id,
12040            error = %err,
12041            "SIGTERM to module failed; the drain wait and kill still apply"
12042        ),
12043    }
12044}
12045
12046/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
12047/// Windows does offer need cooperation this supervisor cannot assume: a console
12048/// control event requires sharing a console with the child, and `WM_CLOSE`
12049/// requires the child to pump a message loop. A supervised server process does
12050/// neither, so there is nothing to send and teardown is the wait followed by the
12051/// kill. Emulating a signal here would mean inventing a stop protocol, which is
12052/// the thing `protocol: "none"` exists to avoid.
12053#[cfg(not(unix))]
12054fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12055    debug!(
12056        module_id,
12057        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12058    );
12059}
12060
12061fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12062    match final_state {
12063        ModuleState::Stopped => TerminalDisposition::Stopped,
12064        ModuleState::Disabled => TerminalDisposition::Disabled,
12065        ModuleState::Restarting => TerminalDisposition::Restarting,
12066        ModuleState::Failed => TerminalDisposition::Failed,
12067        ModuleState::Starting
12068        | ModuleState::Running
12069        | ModuleState::Unresponsive
12070        | ModuleState::Draining => {
12071            unreachable!("terminal exits only finish in terminal or restarting states")
12072        }
12073    }
12074}
12075
12076/// Release a reaped child's registration before allowing another spawn.
12077///
12078/// EOF is not a process-lifetime signal: an inherited socket can stay open
12079/// indefinitely, and serial frame dispatch can be waiting on egress instead of
12080/// reading EOF. After the normal release grace, request connection close (which
12081/// cancels both reads and dispatch), then allow one more release grace for the
12082/// connection guard's forwarding cleanup. Never evict a different connection.
12083async fn release_dead_registration(
12084    registry: &Registry,
12085    forwarding: Option<&ForwardingTable>,
12086    snapshot: &SharedSnapshot,
12087    module_id: &str,
12088    failure: &FailureContext,
12089) -> Result<(), SuperviseError> {
12090    let result = async {
12091        let registration = registry
12092            .get_module(module_id)
12093            .map_err(SuperviseError::Registry)?;
12094        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12095            Ok(()) => return Ok(()),
12096            Err(SuperviseError::RegistrationStillActive { .. }) => {}
12097            Err(err) => return Err(err),
12098        }
12099        let pid = lock_snapshot(snapshot)?.reaped_pid;
12100        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12101            warn!(
12102                module_id,
12103                pid,
12104                connection_id = registration.connection_id.get(),
12105                "reaped module registration outlived release grace; closing dead connection"
12106            );
12107            forwarding.request_connection_close(
12108                registration.connection_id,
12109                CloseReason::new(
12110                    "supervised_process_reaped",
12111                    format!("module '{module_id}' pid {pid} exited"),
12112                ),
12113            );
12114            wait_for_slot_registration_release(
12115                registry,
12116                crate::registry::RegistrationSlot::Connection(registration.connection_id),
12117                REGISTRY_RELEASE_TIMEOUT,
12118            )
12119            .await?;
12120        }
12121        wait_for_registration_release(registry, module_id, Duration::ZERO).await
12122    }
12123    .await;
12124    if let Err(err) = &result {
12125        fail_snapshot(snapshot, Some(module_id), None, failure);
12126        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12127    }
12128    result
12129}
12130
12131/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
12132/// plain stop or restart waits for before it spawns a replacement.
12133async fn wait_for_registration_release(
12134    registry: &Registry,
12135    module_id: &str,
12136    wait: Duration,
12137) -> Result<(), SuperviseError> {
12138    wait_for_slot_registration_release(
12139        registry,
12140        crate::registry::RegistrationSlot::Active(module_id),
12141        wait,
12142    )
12143    .await
12144}
12145
12146/// Wait for the registration in `slot` to go away.
12147///
12148/// Keyed on the slot rather than the bare module id because a successful swap
12149/// never empties the id's active slot (the promoted candidate is in it), so an
12150/// id-keyed wait for the incumbent's release would always time out. Draining a
12151/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
12152/// incumbent's connection instead.
12153async fn wait_for_slot_registration_release(
12154    registry: &Registry,
12155    slot: crate::registry::RegistrationSlot<'_>,
12156    wait: Duration,
12157) -> Result<(), SuperviseError> {
12158    let deadline = Instant::now() + wait;
12159    let mut release_events = registration_release_events().subscribe();
12160    let still_active = |registration: &crate::registry::ModuleRegistration| {
12161        SuperviseError::RegistrationStillActive {
12162            module_id: registration.manifest.module_id.clone(),
12163            waited: wait,
12164        }
12165    };
12166    loop {
12167        let _observed_generation = *release_events.borrow_and_update();
12168        let Some(registration) = registry
12169            .registration(slot)
12170            .map_err(SuperviseError::Registry)?
12171        else {
12172            return Ok(());
12173        };
12174
12175        let now = Instant::now();
12176        if now >= deadline {
12177            return Err(still_active(&registration));
12178        }
12179
12180        let remaining = deadline.saturating_duration_since(now);
12181        match timeout(remaining, release_events.changed()).await {
12182            Ok(Ok(())) | Ok(Err(_)) => {}
12183            Err(_) => return Err(still_active(&registration)),
12184        }
12185    }
12186}
12187
12188#[cfg(test)]
12189mod slot_registration_wait_tests {
12190    use super::*;
12191    use crate::registry::{ConnectionId, RegistrationSlot};
12192    use subc_protocol::manifest::ModuleManifest;
12193
12194    #[tokio::test]
12195    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12196        let registry = Arc::new(Registry::default());
12197        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
12198        let runtime = supervisor.runtime_config();
12199        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12200        let spec = ModuleSpec {
12201            module_id: "enable-stale-registration".to_string(),
12202            program: PathBuf::from("/missing/enable-retry-test"),
12203            args: Vec::new(),
12204            env: Vec::new(),
12205            reserved: false,
12206            reserved_prefixes: Vec::new(),
12207            protocol: ModuleProtocol::Subc,
12208            overlap: Default::default(),
12209            carry_containment: false,
12210        };
12211        let connection = ConnectionId::new(90);
12212        registry
12213            .register_with_control_ops(
12214                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12215                1,
12216                connection,
12217                Vec::new(),
12218            )
12219            .unwrap();
12220        let mut child = None;
12221        let err = set_child_enabled(
12222            &spec,
12223            &runtime,
12224            &registry,
12225            &supervisor.process_liveness,
12226            &snapshot,
12227            &mut child,
12228            true,
12229            &runtime.failure_context(false),
12230        )
12231        .await
12232        .unwrap_err();
12233        assert!(matches!(
12234            err,
12235            SuperviseError::RegistrationStillActive { .. }
12236        ));
12237        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12238        assert!(child.is_none());
12239        registry.deregister_connection(connection).unwrap();
12240        let err = set_child_enabled(
12241            &spec,
12242            &runtime,
12243            &registry,
12244            &supervisor.process_liveness,
12245            &snapshot,
12246            &mut child,
12247            true,
12248            &runtime.failure_context(false),
12249        )
12250        .await
12251        .unwrap_err();
12252        assert!(
12253            matches!(err, SuperviseError::Spawn { .. }),
12254            "second enable must attempt a spawn: {err}"
12255        );
12256        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12257    }
12258
12259    const INCUMBENT: u64 = 1;
12260    const CANDIDATE: u64 = 2;
12261
12262    fn swapped_registry() -> Arc<Registry> {
12263        let registry = Arc::new(Registry::default());
12264        let manifest = ModuleManifest::builder("m", "0.1.0").build();
12265        registry
12266            .register_with_control_ops(
12267                manifest.clone(),
12268                1,
12269                ConnectionId::new(INCUMBENT),
12270                Vec::new(),
12271            )
12272            .unwrap();
12273        registry
12274            .register_candidate_with_control_ops(
12275                manifest,
12276                1,
12277                ConnectionId::new(CANDIDATE),
12278                Vec::new(),
12279            )
12280            .unwrap();
12281        registry
12282    }
12283
12284    /// After a promotion the id's active slot is held by the new process, so an
12285    /// id-keyed wait for the incumbent's release can never succeed; the
12286    /// connection-keyed wait completes as soon as the incumbent deregisters.
12287    #[tokio::test]
12288    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12289        let registry = swapped_registry();
12290        registry.promote_candidate("m").unwrap().unwrap();
12291
12292        assert!(matches!(
12293            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
12294            Err(SuperviseError::RegistrationStillActive { .. })
12295        ));
12296
12297        // Still held while the incumbent's connection has not deregistered.
12298        assert!(matches!(
12299            wait_for_slot_registration_release(
12300                &registry,
12301                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12302                Duration::from_millis(50),
12303            )
12304            .await,
12305            Err(SuperviseError::RegistrationStillActive { .. })
12306        ));
12307
12308        let releaser = Arc::clone(&registry);
12309        let release = tokio::spawn(async move {
12310            sleep(Duration::from_millis(20)).await;
12311            releaser
12312                .deregister_connection(ConnectionId::new(INCUMBENT))
12313                .unwrap();
12314            notify_registration_release();
12315        });
12316        wait_for_slot_registration_release(
12317            &registry,
12318            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12319            Duration::from_secs(5),
12320        )
12321        .await
12322        .expect("the incumbent's own registration is released");
12323        release.await.unwrap();
12324        assert!(registry.get_module("m").unwrap().is_some());
12325    }
12326
12327    /// The candidate slot is waited on separately from the active slot: the
12328    /// incumbent's registration neither holds up nor stands in for it.
12329    #[tokio::test]
12330    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12331        let registry = swapped_registry();
12332        assert!(matches!(
12333            wait_for_slot_registration_release(
12334                &registry,
12335                RegistrationSlot::Candidate("m"),
12336                Duration::from_millis(50),
12337            )
12338            .await,
12339            Err(SuperviseError::RegistrationStillActive { .. })
12340        ));
12341        registry
12342            .deregister_connection(ConnectionId::new(CANDIDATE))
12343            .unwrap();
12344        wait_for_slot_registration_release(
12345            &registry,
12346            RegistrationSlot::Candidate("m"),
12347            Duration::from_millis(50),
12348        )
12349        .await
12350        .expect("a candidate slot with no candidate is released");
12351        assert!(registry
12352            .registration(RegistrationSlot::Active("m"))
12353            .unwrap()
12354            .is_some());
12355    }
12356}
12357
12358fn classify_exit(status: &ExitStatus) -> ExitReport {
12359    ExitReport {
12360        kind: if status.success() {
12361            ExitKind::Clean
12362        } else {
12363            ExitKind::Crash
12364        },
12365        code: status.code(),
12366        signal: exit_signal(status),
12367        at_ms: unix_ms_now(),
12368    }
12369}
12370
12371/// The terminal record for a module whose `wait()` call itself errored (e.g. the
12372/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
12373/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
12374/// disposition still must be `Failed` so the terminal ring is not silently missing
12375/// an entry, matching what `fail_snapshot` records for this same arm.
12376fn wait_error_exit_report() -> ExitReport {
12377    ExitReport {
12378        kind: ExitKind::Crash,
12379        code: None,
12380        signal: None,
12381        at_ms: unix_ms_now(),
12382    }
12383}
12384
12385#[cfg(unix)]
12386fn exit_signal(status: &ExitStatus) -> Option<i32> {
12387    use std::os::unix::process::ExitStatusExt;
12388
12389    status.signal()
12390}
12391
12392#[cfg(not(unix))]
12393fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12394    None
12395}
12396
12397/// Give an operator-touched module its full crash budget back.
12398///
12399/// Named for the counter it used to zero; it now empties the in-window ring,
12400/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
12401/// ledger of what happened survives every operator action.
12402fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12403    update_snapshot(snapshot, Some(module_id), |state| {
12404        state.clear_crash_restarts();
12405    })
12406}
12407
12408fn set_running(
12409    snapshot: &SharedSnapshot,
12410    child: &SupervisedChild,
12411    module_id: &str,
12412    spawn_events: &SpawnEventFeed,
12413) -> Result<(), SuperviseError> {
12414    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12415        module_id: Some(module_id.to_string()),
12416    })?;
12417    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12418    if std::mem::take(&mut state.coalesced_restart_pending) {
12419        let generation = state.spawn_generation;
12420        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12421    }
12422    state.drain_disposition_detail = None;
12423    state.spawn_failure = None;
12424    state.waiting_for_previous_pid = None;
12425    state.waiting_for_previous_detail = None;
12426    // Every caller of this is a plain spawn, which always uses the primary key;
12427    // a promoted swap candidate sets the flag itself after this returns.
12428    state.in_alternate_slot = false;
12429    state.configuration_updated_since_spawn = false;
12430    state.spawned_protocol = Some(child.protocol);
12431    state.state = ModuleState::Running;
12432    state.enabled = true;
12433    state.process_alive = true;
12434    state.pid = child.id();
12435    state.subtree_target = child.subtree_target.clone();
12436    // Probe samples belong to one process. Keep module-level escalation and
12437    // late-answer history, but never charge a new child for its predecessor's misses.
12438    state.health.status = SupervisorHealthStatus::Unknown;
12439    state.health.consecutive_failures = 0;
12440    state.health.detail = None;
12441    state.health.metrics = None;
12442    state.health.last_probe_ms = None;
12443    state.failed_retry.healthy_since = None;
12444    state.failed_retry.failed_at = None;
12445    state.failed_retry.running_since = Some(child.spawned_at);
12446    #[cfg(windows)]
12447    {
12448        state.pipe_consumed = child
12449            .nonce_delivery
12450            .as_ref()
12451            .map(|delivery| delivery.consumed());
12452    }
12453    #[cfg(target_os = "macos")]
12454    {
12455        state.report_ready = Some(Arc::clone(&child.report_ready));
12456    }
12457    state.spawned_at_ms = Some(child.spawned_at_ms);
12458    state.exec_outcome = Some(Arc::clone(&child.exec.0));
12459    state.spawned_from = Some(child.spawned_from.clone());
12460    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12461    state.spawned_file_identity = child.spawned_file_identity.clone();
12462    state.process_start_time = child.process_start_time;
12463    Ok(())
12464}
12465
12466fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12467    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12468    state.failed_retry.healthy_since = None;
12469    state.failed_retry.running_since = None;
12470    state.waiting_for_previous_pid = None;
12471    state.waiting_for_previous_detail = None;
12472    state.process_alive = false;
12473    state.spawned_protocol = None;
12474    state.pid = None;
12475    #[cfg(windows)]
12476    {
12477        state.pipe_consumed = None;
12478    }
12479    #[cfg(target_os = "macos")]
12480    {
12481        state.report_ready = None;
12482    }
12483    state.spawned_at_ms = None;
12484    state.exec_outcome = None;
12485    state.spawned_from = None;
12486    state.launch_spec_digest = None;
12487    state.spawned_file_identity = None;
12488    state.process_start_time = None;
12489    state.deliberate_severance = None;
12490}
12491
12492#[cfg(test)]
12493fn record_deliberate_severance(
12494    snapshot: &SharedSnapshot,
12495    identity: ProcessIdentity,
12496) -> Result<(), SuperviseError> {
12497    update_snapshot(snapshot, None, |state| {
12498        state.deliberate_severance = Some(identity);
12499    })
12500}
12501
12502fn apply_deliberate_severance_marker(
12503    snapshot: &SharedSnapshot,
12504    exited_identity: Option<ProcessIdentity>,
12505    mut exit_report: ExitReport,
12506) -> ExitReport {
12507    let marker = lock_snapshot(snapshot)
12508        .ok()
12509        .and_then(|mut state| state.deliberate_severance.take());
12510    if marker.is_some() && marker == exited_identity {
12511        exit_report.kind = ExitKind::DeliberateSeverance;
12512    }
12513    exit_report
12514}
12515
12516fn classify_reaped_child_exit(
12517    snapshot: &SharedSnapshot,
12518    child: &SupervisedChild,
12519    status: &ExitStatus,
12520) -> ExitReport {
12521    let _ = update_snapshot(snapshot, None, |state| {
12522        state.reaped_pid = Some(child.pid);
12523        state.spawn_failure = child.spawn_failure.clone();
12524    });
12525    if child.spawn_failure.as_deref() == Some("registration_timeout") {
12526        registration_failure_exit_report(classify_exit(status))
12527    } else {
12528        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12529    }
12530}
12531
12532fn fail_snapshot(
12533    snapshot: &SharedSnapshot,
12534    module_id: Option<&str>,
12535    last_exit: Option<ExitReport>,
12536    failure: &FailureContext,
12537) {
12538    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12539        state.state = ModuleState::Failed;
12540        clear_current_process_facts(state);
12541        if failure.may_retry() && state.enabled {
12542            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12543        } else if failure.shutdown.is_set() {
12544            state.failed_retry.cancel();
12545        }
12546        if let Some(last_exit) = last_exit {
12547            state.last_exit = Some(last_exit);
12548        }
12549    }) {
12550        error!(error = %err, "failed to mark supervisor state failed");
12551    }
12552}
12553
12554fn update_snapshot(
12555    snapshot: &SharedSnapshot,
12556    module_id: Option<&str>,
12557    update: impl FnOnce(&mut SupervisorSnapshot),
12558) -> Result<(), SuperviseError> {
12559    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12560        module_id: module_id.map(ToOwned::to_owned),
12561    })?;
12562    update(&mut state);
12563    Ok(())
12564}
12565
12566const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12567
12568fn lock_snapshot_for_control<'a>(
12569    snapshot: &'a SharedSnapshot,
12570    module_id: &str,
12571    caller: &'static str,
12572) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12573    let started_at = Instant::now();
12574    let guard = lock_snapshot(snapshot)?;
12575    let waited = started_at.elapsed();
12576    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12577        warn!(
12578            module_id = %module_id,
12579            waited_ms = waited.as_millis() as u64,
12580            caller = %caller,
12581            "slow snapshot lock"
12582        );
12583    }
12584    Ok(guard)
12585}
12586
12587fn lock_snapshot(
12588    snapshot: &SharedSnapshot,
12589) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12590    snapshot
12591        .lock()
12592        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12593}
12594
12595#[cfg(test)]
12596mod terminal_history_tests {
12597    use std::{
12598        path::PathBuf,
12599        sync::Arc,
12600        time::{Duration, Instant},
12601    };
12602
12603    use tokio::{sync::mpsc, time::sleep};
12604
12605    use super::{
12606        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12607        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12608        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12609        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12610        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12611        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12612        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12613        SupervisorHealthStatus, SupervisorSnapshot,
12614    };
12615    // The supervisor's clock, distinct from the `std::time::Instant` these tests
12616    // use for their own wall-clock deadlines: crash-restart instants must be on
12617    // the same clock the production code stamps them with, which is tokio's (and
12618    // is what `start_paused` tests can move).
12619    use super::Instant as ClockInstant;
12620    use crate::{
12621        registry::Registry,
12622        terminal_ring::{TerminalRing, TerminalRingConfig},
12623    };
12624    use std::sync::Mutex;
12625    use subc_control::TerminalDisposition;
12626
12627    /// See the twin in `control.rs` for why this derives the path from
12628    /// `current_exe()` and why the existence check is here: `--lib` alone does
12629    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
12630    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
12631    pub(super) fn fake_aft_stub_path() -> PathBuf {
12632        let mut path = std::env::current_exe().expect("current_exe available in tests");
12633        path.pop();
12634        path.pop();
12635        path.push(if cfg!(windows) {
12636            "fake-aft-stub.exe"
12637        } else {
12638            "fake-aft-stub"
12639        });
12640        assert!(
12641            path.exists(),
12642            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12643             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12644            path.display()
12645        );
12646        path
12647    }
12648
12649    #[test]
12650    fn reserved_never_spawned_refuses_every_hello() {
12651        // The canary hole: a reserved id whose module has never spawned had NO
12652        // gate entry and admitted anyone -- the reservation protected the nonce
12653        // holder, not the NAME. Now the entry is present with no legitimate
12654        // holder and refuses all comers.
12655        let supervisor = SupervisorHandle::default();
12656        supervisor.apply_identity_configuration(&ModuleSpec {
12657            module_id: "never-spawned".to_string(),
12658            program: PathBuf::from("/usr/bin/false"),
12659            args: Vec::new(),
12660            env: Vec::new(),
12661            reserved: true,
12662            reserved_prefixes: Vec::new(),
12663            protocol: ModuleProtocol::Subc,
12664            overlap: Default::default(),
12665            carry_containment: false,
12666        });
12667        assert!(
12668            supervisor
12669                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12670                .is_err(),
12671            "forged nonce must refuse on a reserved never-spawned id"
12672        );
12673        assert!(
12674            supervisor
12675                .reserved_hello_verification("never-spawned", None)
12676                .is_err(),
12677            "absent nonce must refuse on a reserved never-spawned id"
12678        );
12679        // And a real spawn nonce minted later admits exactly that nonce.
12680        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12681        supervisor.apply_identity_configuration(&ModuleSpec {
12682            module_id: "never-spawned".to_string(),
12683            program: PathBuf::from("/usr/bin/false"),
12684            args: Vec::new(),
12685            env: Vec::new(),
12686            reserved: true,
12687            reserved_prefixes: Vec::new(),
12688            protocol: ModuleProtocol::Subc,
12689            overlap: Default::default(),
12690            carry_containment: false,
12691        });
12692        assert!(supervisor
12693            .reserved_hello_verification("never-spawned", Some("minted"))
12694            .is_ok());
12695        assert!(supervisor
12696            .reserved_hello_verification("never-spawned", Some("forged"))
12697            .is_err());
12698    }
12699
12700    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12701    /// happened, which is what "spent budget" looks like to every reader.
12702    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12703        let now = ClockInstant::now();
12704        for _ in 0..count {
12705            state.crash_restarts.push_back(now);
12706        }
12707    }
12708
12709    /// Age the oldest recorded restart out of `window`, standing in for the hours
12710    /// that would otherwise have to pass. Injecting the instant is the point: a
12711    /// test that slept a real window would take ten minutes and still prove less.
12712    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12713        let aged = state
12714            .crash_restarts
12715            .front()
12716            .expect("a crash restart must be recorded before it can be aged")
12717            .checked_sub(window + Duration::from_secs(1))
12718            .expect("the test clock is far enough from its origin to age an instant");
12719        state.crash_restarts[0] = aged;
12720    }
12721
12722    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12723        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12724        seed_crash_restarts(&mut state, count);
12725        state
12726    }
12727
12728    #[test]
12729    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12730        let policy = RestartPolicy::new(3, Duration::ZERO);
12731        let now = ClockInstant::now();
12732        assert!(daemon_will_restart(
12733            &mut snapshot_with_restarts(true, 2),
12734            &policy,
12735            now
12736        ));
12737        assert!(!daemon_will_restart(
12738            &mut snapshot_with_restarts(true, 3),
12739            &policy,
12740            now
12741        ));
12742        assert!(!daemon_will_restart(
12743            &mut snapshot_with_restarts(false, 0),
12744            &policy,
12745            now
12746        ));
12747    }
12748
12749    #[test]
12750    fn crash_restart_backoff_escalates_with_in_window_count() {
12751        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12752            .with_max_backoff(Duration::from_secs(30));
12753        let now = ClockInstant::now();
12754        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12755        let schedules = (0..4)
12756            .map(|_| {
12757                state
12758                    .next_crash_restart(&policy, now)
12759                    .expect("the test policy allows four crash restarts")
12760            })
12761            .collect::<Vec<_>>();
12762
12763        assert_eq!(
12764            schedules
12765                .iter()
12766                .map(|schedule| schedule.restart_in_window)
12767                .collect::<Vec<_>>(),
12768            vec![0, 1, 2, 3]
12769        );
12770        assert_eq!(
12771            schedules
12772                .iter()
12773                .map(|schedule| schedule.delay)
12774                .collect::<Vec<_>>(),
12775            vec![
12776                Duration::from_millis(100),
12777                Duration::from_secs(1),
12778                Duration::from_secs(10),
12779                Duration::from_secs(30),
12780            ]
12781        );
12782    }
12783
12784    #[test]
12785    fn crash_restart_backoff_resets_after_ring_clear() {
12786        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12787        let now = ClockInstant::now();
12788        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12789        assert_eq!(
12790            state.next_crash_restart(&policy, now).unwrap().delay,
12791            Duration::from_millis(100)
12792        );
12793        assert_eq!(
12794            state.next_crash_restart(&policy, now).unwrap().delay,
12795            Duration::from_secs(1)
12796        );
12797
12798        state.clear_crash_restarts();
12799        let schedule = state
12800            .next_crash_restart(&policy, now)
12801            .expect("a cleared ring must allow another restart");
12802        assert_eq!(schedule.restart_in_window, 0);
12803        assert_eq!(schedule.delay, Duration::from_millis(100));
12804    }
12805
12806    #[test]
12807    fn crash_restart_backoff_ignores_aged_restarts() {
12808        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12809        let now = ClockInstant::now();
12810        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12811        state
12812            .next_crash_restart(&policy, now)
12813            .expect("the first restart is allowed");
12814        state
12815            .next_crash_restart(&policy, now)
12816            .expect("the second restart is allowed");
12817        state.crash_restarts[0] = now
12818            .checked_sub(policy.window + Duration::from_secs(1))
12819            .expect("the fake clock can age a restart past the window");
12820
12821        let schedule = state
12822            .next_crash_restart(&policy, now)
12823            .expect("an aged restart must release its slot");
12824        assert_eq!(schedule.restart_in_window, 1);
12825        assert_eq!(schedule.delay, Duration::from_secs(1));
12826        assert_eq!(state.crash_restarts.len(), 2);
12827    }
12828
12829    /// The budget is a rate: the same three spent restarts refuse a respawn
12830    /// while they are recent and allow one once they have aged past the window.
12831    /// Nothing about the module changed in between, which is the whole point.
12832    #[test]
12833    fn a_budget_spent_before_the_window_no_longer_refuses() {
12834        let policy = RestartPolicy::new(3, Duration::ZERO);
12835        let mut state = snapshot_with_restarts(true, 3);
12836        let now = ClockInstant::now();
12837        assert!(!daemon_will_restart(&mut state, &policy, now));
12838
12839        assert!(daemon_will_restart(
12840            &mut state,
12841            &policy,
12842            now + policy.window + Duration::from_secs(1)
12843        ));
12844        assert!(
12845            state.crash_restarts.is_empty(),
12846            "reading the budget must drop the instants that left the window"
12847        );
12848    }
12849
12850    fn module_with_recovery_snapshot(
12851        state: ModuleState,
12852        enabled: bool,
12853        restart_count: u32,
12854    ) -> SupervisedModule {
12855        let registry = Arc::new(Registry::default());
12856        let supervisor =
12857            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12858        let runtime = supervisor.runtime_config();
12859        let spec = ModuleSpec {
12860            module_id: "recovery-snapshot".to_string(),
12861            program: fake_aft_stub_path(),
12862            args: Vec::new(),
12863            env: Vec::new(),
12864            reserved: false,
12865            reserved_prefixes: Vec::new(),
12866            protocol: ModuleProtocol::Subc,
12867            overlap: Default::default(),
12868            carry_containment: false,
12869        };
12870        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12871        seed_crash_restarts(&mut snapshot, restart_count);
12872        // These tests read synthetic snapshots. A real child and monitor would
12873        // race those reads by replacing the requested state during startup.
12874        let (commands, _rx) = mpsc::channel(4);
12875        SupervisedModule {
12876            inner: Arc::new(SupervisedModuleInner {
12877                module_id: spec.module_id.clone(),
12878                registry,
12879                snapshot: Arc::new(Mutex::new(snapshot)),
12880                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12881                    spec,
12882                    health: runtime.health,
12883                })),
12884                stderr_ring: runtime.stderr_ring,
12885                terminal_ring: runtime.terminal_ring,
12886                commands,
12887                monitor: Mutex::new(None),
12888                restart_policy: runtime.restart_policy,
12889                effective_drain_timeout: runtime.effective_drain_timeout,
12890                provenance_probe: supervisor.provenance_probe.clone(),
12891                retained: runtime.child_roster.retained_view(),
12892            }),
12893        }
12894    }
12895
12896    #[cfg(target_os = "linux")]
12897    #[tokio::test]
12898    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12899        let supervisor =
12900            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12901                .with_cgroup_placement(None);
12902        let result = supervisor.spawn(ModuleSpec {
12903            module_id: "no-cgroup-placement".to_string(),
12904            program: fake_aft_stub_path(),
12905            args: Vec::new(),
12906            env: Vec::new(),
12907            reserved: false,
12908            reserved_prefixes: Vec::new(),
12909            protocol: ModuleProtocol::Subc,
12910            overlap: Default::default(),
12911            carry_containment: false,
12912        });
12913
12914        assert!(
12915            result.is_ok(),
12916            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12917        );
12918    }
12919
12920    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12921    async fn undecided_snapshot_uses_shared_restart_predicate() {
12922        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12923            .will_recover_after_connection_loss()
12924            .unwrap());
12925        assert!(
12926            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12927                .will_recover_after_connection_loss()
12928                .unwrap()
12929        );
12930    }
12931
12932    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12933    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12934        assert!(
12935            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12936                .will_recover_after_connection_loss()
12937                .unwrap()
12938        );
12939    }
12940
12941    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12942    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12943        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12944            .will_recover_after_connection_loss()
12945            .unwrap());
12946        assert!(
12947            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12948                .will_recover_after_connection_loss()
12949                .unwrap()
12950        );
12951    }
12952
12953    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12954    async fn warming_snapshot_is_limited_to_startup_phases() {
12955        for state in [
12956            ModuleState::Starting,
12957            ModuleState::Running,
12958            ModuleState::Restarting,
12959        ] {
12960            assert!(
12961                module_with_recovery_snapshot(state, true, 0)
12962                    .is_warming()
12963                    .unwrap(),
12964                "{state:?} should be warming"
12965            );
12966        }
12967        for state in [
12968            ModuleState::Unresponsive,
12969            ModuleState::Draining,
12970            ModuleState::Stopped,
12971            ModuleState::Failed,
12972            ModuleState::Disabled,
12973        ] {
12974            assert!(
12975                !module_with_recovery_snapshot(state, true, 0)
12976                    .is_warming()
12977                    .unwrap(),
12978                "{state:?} should not be warming"
12979            );
12980        }
12981    }
12982
12983    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12984    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12985        let registry = Arc::new(Registry::default());
12986        let supervisor =
12987            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12988        let module = supervisor
12989            .spawn(ModuleSpec {
12990                module_id: "terminal-history".to_string(),
12991                program: fake_aft_stub_path(),
12992                args: Vec::new(),
12993                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12994                reserved: false,
12995                reserved_prefixes: Vec::new(),
12996                protocol: ModuleProtocol::Subc,
12997                overlap: Default::default(),
12998                carry_containment: false,
12999            })
13000            .unwrap();
13001
13002        let deadline = Instant::now() + Duration::from_secs(5);
13003        loop {
13004            let history = module.terminal_history();
13005            if history.entries.len() == 2 {
13006                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
13007                assert_eq!(history.dropped, 0);
13008                assert_eq!(
13009                    history
13010                        .entries
13011                        .iter()
13012                        .map(|entry| entry.exit_code)
13013                        .collect::<Vec<_>>(),
13014                    vec![Some(23), Some(23)]
13015                );
13016                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13017                return;
13018            }
13019            assert!(
13020                Instant::now() < deadline,
13021                "module did not retain two terminal exits: {history:?}"
13022            );
13023            sleep(Duration::from_millis(10)).await;
13024        }
13025    }
13026
13027    /// A disable issued while a crash respawn is still backing off must preempt
13028    /// that respawn: the operator's stop wins, the disable must not queue behind
13029    /// the backoff, and the module must never come back up afterwards.
13030    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13031    async fn disable_during_crash_backoff_cancels_pending_respawn() {
13032        let backoff = Duration::from_secs(2);
13033        let supervisor = Supervisor::new_for_test(
13034            Arc::new(Registry::default()),
13035            RestartPolicy::new(10, backoff),
13036        );
13037        let module = supervisor
13038            .spawn(ModuleSpec {
13039                module_id: "disable-during-backoff".to_string(),
13040                program: fake_aft_stub_path(),
13041                args: Vec::new(),
13042                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13043                reserved: false,
13044                reserved_prefixes: Vec::new(),
13045                protocol: ModuleProtocol::Subc,
13046                overlap: Default::default(),
13047                carry_containment: false,
13048            })
13049            .unwrap();
13050
13051        // Wait for the first crash to put the module into its backoff window.
13052        let deadline = Instant::now() + Duration::from_secs(5);
13053        loop {
13054            if module.status().unwrap().state == ModuleState::Restarting {
13055                break;
13056            }
13057            assert!(
13058                Instant::now() < deadline,
13059                "module never entered the crash backoff"
13060            );
13061            sleep(Duration::from_millis(10)).await;
13062        }
13063
13064        let started = Instant::now();
13065        module.set_enabled(false).await.unwrap();
13066        let waited = started.elapsed();
13067
13068        assert!(
13069            waited < backoff / 2,
13070            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13071        );
13072        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13073
13074        // Outlast the backoff: the respawn it was counting down to must never run.
13075        sleep(backoff + Duration::from_millis(500)).await;
13076        let status = module.status().unwrap();
13077        assert_eq!(status.state, ModuleState::Disabled);
13078        assert_eq!(
13079            status.spawn_generation, 1,
13080            "module respawned after the operator disabled it"
13081        );
13082    }
13083
13084    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
13085    /// the shape of nats-server, the program this rule exists for.
13086    #[cfg(unix)]
13087    fn protocol_none_sigterm_exits_clean_spec(
13088        module_id: &str,
13089        dir: &std::path::Path,
13090    ) -> (ModuleSpec, PathBuf, PathBuf) {
13091        let ready = dir.join("ready");
13092        let marker = dir.join("sigterm");
13093        let spec = ModuleSpec {
13094            module_id: module_id.to_string(),
13095            program: fake_aft_stub_path(),
13096            args: Vec::new(),
13097            env: vec![
13098                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13099                (
13100                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13101                    marker.display().to_string(),
13102                ),
13103                (
13104                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13105                    ready.display().to_string(),
13106                ),
13107            ],
13108            reserved: false,
13109            reserved_prefixes: Vec::new(),
13110            protocol: ModuleProtocol::None,
13111            overlap: Default::default(),
13112            carry_containment: false,
13113        };
13114        (spec, ready, marker)
13115    }
13116
13117    /// Wait for a file the child writes, so a signal is never sent before the
13118    /// child's SIGTERM handler is installed (the default disposition would
13119    /// kill it by signal and the exit would not be clean).
13120    #[cfg(unix)]
13121    async fn wait_for_file(path: &std::path::Path) {
13122        let deadline = Instant::now() + Duration::from_secs(10);
13123        while !path.exists() {
13124            assert!(
13125                Instant::now() < deadline,
13126                "{} never appeared",
13127                path.display()
13128            );
13129            sleep(Duration::from_millis(10)).await;
13130        }
13131    }
13132
13133    /// A protocol-none module that exits 0 because something OUTSIDE the
13134    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
13135    /// the crash-path disposition rather than `stopped`.
13136    #[cfg(unix)]
13137    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13138    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13139        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13140        let (spec, ready, marker) =
13141            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13142        let supervisor = Supervisor::new_for_test(
13143            Arc::new(Registry::default()),
13144            RestartPolicy::new(3, Duration::ZERO),
13145        );
13146        let module = supervisor.spawn(spec).unwrap();
13147        wait_for_file(&ready).await;
13148        // The ready file proves the child installed its SIGTERM handler, not
13149        // that the supervisor has processed the privacy trampoline's exec
13150        // acknowledgement. On macOS status withholds the pid until then.
13151        let deadline = Instant::now() + Duration::from_secs(10);
13152        let first_pid = loop {
13153            let status = module.status().unwrap();
13154            if status.state == ModuleState::Running {
13155                if let Some(pid) = status.pid {
13156                    break pid;
13157                }
13158            }
13159            assert!(
13160                Instant::now() < deadline,
13161                "a running module must report its pid after exec confirmation: {status:?}"
13162            );
13163            sleep(Duration::from_millis(10)).await;
13164        };
13165
13166        rustix::process::kill_process(
13167            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13168            rustix::process::Signal::TERM,
13169        )
13170        .unwrap();
13171
13172        let deadline = Instant::now() + Duration::from_secs(10);
13173        let respawned = loop {
13174            let status = module.status().unwrap();
13175            if status.state == ModuleState::Running
13176                && status.pid.is_some_and(|pid| pid != first_pid)
13177            {
13178                break status;
13179            }
13180            assert!(
13181                Instant::now() < deadline,
13182                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13183            );
13184            sleep(Duration::from_millis(10)).await;
13185        };
13186        assert_eq!(respawned.spawn_generation, 2);
13187        assert!(
13188            marker.exists(),
13189            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13190        );
13191
13192        let history = module.terminal_history();
13193        assert_eq!(history.entries.len(), 1, "{history:?}");
13194        let entry = &history.entries[0];
13195        assert_eq!(entry.exit_code, Some(0));
13196        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13197        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13198
13199        module.stop().await.unwrap();
13200    }
13201
13202    /// Repeated unrequested clean exits of a protocol-none module spend the
13203    /// restart budget exactly as crashes do, and the module ends `failed` with
13204    /// the budget named.
13205    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13206    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13207        let supervisor = Supervisor::new_for_test(
13208            Arc::new(Registry::default()),
13209            RestartPolicy::new(1, Duration::ZERO),
13210        );
13211        let module = supervisor
13212            .spawn(ModuleSpec {
13213                module_id: "none-clean-exit-budget".to_string(),
13214                program: fake_aft_stub_path(),
13215                args: Vec::new(),
13216                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13217                reserved: false,
13218                reserved_prefixes: Vec::new(),
13219                protocol: ModuleProtocol::None,
13220                overlap: Default::default(),
13221                carry_containment: false,
13222            })
13223            .unwrap();
13224
13225        // Failed follows the terminal write; this deadline only bounds a hang,
13226        // not an assumed duration for the two launches or their exit recording.
13227        let deadline = Instant::now() + Duration::from_secs(10);
13228        loop {
13229            let status = module.status().unwrap();
13230            if status.state == ModuleState::Failed {
13231                break;
13232            }
13233            assert!(
13234                Instant::now() < deadline,
13235                "module never exhausted its budget: {status:?} {:?}",
13236                module.terminal_history()
13237            );
13238            sleep(Duration::from_millis(10)).await;
13239        }
13240        let history = module.terminal_history();
13241        assert_eq!(
13242            history
13243                .entries
13244                .iter()
13245                .map(|entry| (entry.exit_code, entry.disposition.clone()))
13246                .collect::<Vec<_>>(),
13247            vec![
13248                (Some(0), TerminalDisposition::Restarting),
13249                (Some(0), TerminalDisposition::Failed),
13250            ]
13251        );
13252        let detail = history.entries[1]
13253            .disposition_detail
13254            .as_deref()
13255            .expect("a budget failure names the budget");
13256        assert!(detail.contains("max_restarts=1"), "{detail}");
13257        assert_eq!(module.status().unwrap().spawn_generation, 2);
13258    }
13259
13260    #[test]
13261    fn restart_budget_failure_is_published_after_its_terminal_record() {
13262        use std::io::{Read, Write};
13263        let supervisor = Supervisor::new_for_test(
13264            Arc::new(Registry::default()),
13265            RestartPolicy::new(0, Duration::ZERO),
13266        );
13267        let runtime = supervisor.runtime_config();
13268        let spec = ModuleSpec {
13269            protocol: ModuleProtocol::None,
13270            ..windowed_crash_spec("budget-publication-order")
13271        };
13272        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13273            ModuleState::Running,
13274            true,
13275        )));
13276        lock_snapshot(&snapshot).unwrap().process_alive = true;
13277        let reaped_snapshot = snapshot.clone();
13278        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13279        let history = runtime.terminal_ring.clone();
13280        let reap = std::thread::spawn(move || {
13281            tokio::runtime::Builder::new_current_thread()
13282                .enable_all()
13283                .build()
13284                .unwrap()
13285                .block_on(on_child_exit(
13286                    &spec,
13287                    runtime.restart_policy,
13288                    &supervisor.registry,
13289                    &reaped_snapshot,
13290                    &runtime.terminal_ring,
13291                    &runtime.spawn_events,
13292                    &runtime.child_roster,
13293                    ExitReport {
13294                        kind: ExitKind::Clean,
13295                        code: Some(0),
13296                        signal: None,
13297                        at_ms: 1,
13298                    },
13299                    true,
13300                ))
13301        });
13302        let mut reached = [0];
13303        barrier.read_exact(&mut reached).unwrap();
13304        assert_eq!(&reached, b"R");
13305        let (before_state, before_alive) = {
13306            let state = lock_snapshot(&snapshot).unwrap();
13307            (state.state, state.process_alive)
13308        };
13309        let before_history = history.lock().unwrap().snapshot();
13310        barrier.write_all(b"X").unwrap();
13311        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13312        assert!(before_history.entries.is_empty());
13313        assert_ne!(
13314            before_state,
13315            ModuleState::Failed,
13316            "Failed was visible before its terminal record could be written"
13317        );
13318        assert!(before_alive);
13319        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13320        let history = history.lock().unwrap().snapshot();
13321        assert_eq!(history.entries.len(), 1);
13322        assert_eq!(history.entries[0].exit_code, Some(0));
13323        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13324    }
13325
13326    #[test]
13327    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13328        use std::io::{Read, Write};
13329
13330        let supervisor = Supervisor::new_for_test(
13331            Arc::new(Registry::default()),
13332            RestartPolicy::new(3, Duration::ZERO),
13333        );
13334        let mut runtime = supervisor.runtime_config();
13335        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13336        let snapshot = module.inner.snapshot.clone();
13337        lock_snapshot(&snapshot).unwrap().process_alive = true;
13338        runtime.terminal_ring = module.inner.terminal_ring.clone();
13339        let spec = windowed_crash_spec(module.module_id());
13340        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13341        let reaped_snapshot = snapshot.clone();
13342        let reap = std::thread::spawn(move || {
13343            tokio::runtime::Builder::new_current_thread()
13344                .enable_all()
13345                .build()
13346                .unwrap()
13347                .block_on(on_child_exit(
13348                    &spec,
13349                    runtime.restart_policy,
13350                    &supervisor.registry,
13351                    &reaped_snapshot,
13352                    &runtime.terminal_ring,
13353                    &runtime.spawn_events,
13354                    &runtime.child_roster,
13355                    ExitReport {
13356                        kind: ExitKind::Crash,
13357                        code: Some(2),
13358                        signal: None,
13359                        at_ms: 1,
13360                    },
13361                    true,
13362                ))
13363        });
13364
13365        // Hold the crash handler between its budget decision and publication.
13366        // Connection cleanup must still see the recovery that this crash chose,
13367        // even when that recovery consumes the last available restart slot.
13368        let mut reached = [0];
13369        barrier.read_exact(&mut reached).unwrap();
13370        assert_eq!(&reached, b"R");
13371        let before_status = module.status().unwrap();
13372        let before_history = module.terminal_history();
13373        let before_recovers = module.will_recover_after_connection_loss().unwrap();
13374        barrier.write_all(b"X").unwrap();
13375        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13376
13377        assert_eq!(before_status.state, ModuleState::Running);
13378        assert!(before_status.process_alive);
13379        assert_eq!(before_status.restart_count, 3);
13380        assert!(before_history.entries.is_empty());
13381        assert!(
13382            before_recovers,
13383            "connection cleanup would report a terminal close for a scheduled crash restart"
13384        );
13385        let restarting = module.status().unwrap();
13386        assert_eq!(restarting.state, ModuleState::Restarting);
13387        assert_eq!(restarting.restart_count, 3);
13388        assert!(module.will_recover_after_connection_loss().unwrap());
13389        let history = module.terminal_history();
13390        assert_eq!(history.entries.len(), 1);
13391        assert_eq!(
13392            history.entries[0].disposition,
13393            TerminalDisposition::Restarting
13394        );
13395
13396        // Once the replacement is running, its next crash must be judged
13397        // against the exhausted budget rather than the previous recovery.
13398        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13399            state.state = ModuleState::Running;
13400            state.respawn_pending = false;
13401        })
13402        .unwrap();
13403        assert!(!module.will_recover_after_connection_loss().unwrap());
13404        update_snapshot(&snapshot, Some(module.module_id()), |state| {
13405            state.enabled = false;
13406            state.respawn_pending = true;
13407        })
13408        .unwrap();
13409        assert!(!module.will_recover_after_connection_loss().unwrap());
13410    }
13411
13412    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13413        use std::net::{TcpListener, TcpStream};
13414        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13415        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13416        let (barrier, _) = listener.accept().unwrap();
13417        for stream in [&writer, &barrier] {
13418            stream
13419                .set_read_timeout(Some(Duration::from_secs(10)))
13420                .unwrap();
13421            stream
13422                .set_write_timeout(Some(Duration::from_secs(10)))
13423                .unwrap();
13424        }
13425        *events.1.lock().unwrap() = Some(writer);
13426        barrier
13427    }
13428
13429    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13430        use std::io::{Read, Write};
13431        let barrier = events.1.lock().unwrap().take();
13432        if let Some(mut barrier) = barrier {
13433            barrier.write_all(b"R").unwrap();
13434            let mut release = [0];
13435            barrier.read_exact(&mut release).unwrap();
13436            assert_eq!(&release, b"X");
13437        }
13438    }
13439
13440    #[test]
13441    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13442        use std::io::{Read, Write};
13443
13444        let supervisor = Supervisor::new_for_test(
13445            Arc::new(Registry::default()),
13446            RestartPolicy::new(0, Duration::ZERO),
13447        );
13448        let mut runtime = supervisor.runtime_config();
13449        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13450        let snapshot = module.inner.snapshot.clone();
13451        runtime.terminal_ring = module.inner.terminal_ring.clone();
13452        let spec = ModuleSpec {
13453            module_id: module.module_id().into(),
13454            program: fake_aft_stub_path(),
13455            protocol: ModuleProtocol::None,
13456            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13457            ..windowed_crash_spec(module.module_id())
13458        };
13459        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13460        let reap = std::thread::spawn(move || {
13461            tokio::runtime::Builder::new_current_thread()
13462                .enable_all()
13463                .build()
13464                .unwrap()
13465                .block_on(async {
13466                    let mut child =
13467                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13468                    health_restart_child(
13469                        &spec,
13470                        &runtime,
13471                        &supervisor.registry,
13472                        &supervisor.process_liveness,
13473                        &snapshot,
13474                        &mut child,
13475                        SupervisorHealthStatus::Failing,
13476                        Some("health probe did not answer"),
13477                        1,
13478                    )
13479                    .await
13480                    .unwrap();
13481                    assert!(child.is_none());
13482                });
13483        });
13484        // The loopback barrier acknowledges entry into terminal recording by
13485        // health_restart_child's drain, then waits for the test's release byte.
13486        // Status and history are observed while that writer is held, not after
13487        // an elapsed-time guess about when the health action has completed.
13488        let mut reached = [0];
13489        barrier.read_exact(&mut reached).unwrap();
13490        assert_eq!(&reached, b"R");
13491        let before_status = module.status().unwrap();
13492        let before_history = module.terminal_history();
13493        barrier.write_all(b"X").unwrap();
13494        reap.join().unwrap();
13495        assert!(before_history.entries.is_empty());
13496        assert_ne!(
13497            before_status.state,
13498            ModuleState::Failed,
13499            "Failed was visible before its terminal record could be written"
13500        );
13501        assert!(
13502            before_status.process_alive,
13503            "process death was visible before its terminal record could be written"
13504        );
13505        let failed = module.status().unwrap();
13506        assert_eq!(failed.state, ModuleState::Failed);
13507        assert!(!failed.process_alive);
13508        let history = module.terminal_history();
13509        assert_eq!(history.entries.len(), 1);
13510        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13511        assert_eq!(
13512            history.entries[0].disposition_detail.as_deref(),
13513            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13514        );
13515    }
13516
13517    #[test]
13518    fn observed_exits_publish_process_death_after_terminal_history() {
13519        use std::io::{Read, Write};
13520        for (kind, enabled, shutdown, wait_error, expected_state) in [
13521            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13522            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13523            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13524            (
13525                ExitKind::DeliberateSeverance,
13526                true,
13527                false,
13528                false,
13529                ModuleState::Restarting,
13530            ),
13531            #[cfg(unix)]
13532            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13533            (ExitKind::Crash, true, false, true, ModuleState::Failed),
13534        ] {
13535            let supervisor = Supervisor::new_for_test(
13536                Arc::new(Registry::default()),
13537                RestartPolicy::new(1, Duration::ZERO),
13538            );
13539            let runtime = supervisor.runtime_config();
13540            #[cfg(unix)]
13541            if shutdown {
13542                runtime.child_roster.close();
13543            }
13544            #[cfg(not(unix))]
13545            let _ = shutdown;
13546            let spec = windowed_crash_spec("exit-publication-order");
13547            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13548                ModuleState::Running,
13549                enabled,
13550            )));
13551            lock_snapshot(&snapshot).unwrap().process_alive = true;
13552            let reaped_snapshot = snapshot.clone();
13553            let history = runtime.terminal_ring.clone();
13554            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13555            let reap = std::thread::spawn(move || {
13556                if wait_error {
13557                    super::fail_after_wait_error(
13558                        &spec.module_id,
13559                        &reaped_snapshot,
13560                        &runtime.terminal_ring,
13561                        &runtime.spawn_events,
13562                        "wait failed".into(),
13563                        &runtime.failure_context(true),
13564                    );
13565                } else {
13566                    tokio::runtime::Builder::new_current_thread()
13567                        .enable_all()
13568                        .build()
13569                        .unwrap()
13570                        .block_on(on_child_exit(
13571                            &spec,
13572                            runtime.restart_policy,
13573                            &supervisor.registry,
13574                            &reaped_snapshot,
13575                            &runtime.terminal_ring,
13576                            &runtime.spawn_events,
13577                            &runtime.child_roster,
13578                            ExitReport {
13579                                kind,
13580                                code: Some(0),
13581                                signal: None,
13582                                at_ms: 1,
13583                            },
13584                            true,
13585                        ));
13586                }
13587            });
13588            let mut reached = [0];
13589            barrier.read_exact(&mut reached).unwrap();
13590            assert_eq!(&reached, b"R");
13591            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13592            let before_history = history.lock().unwrap().snapshot();
13593            barrier.write_all(b"X").unwrap();
13594            reap.join().unwrap();
13595            assert!(
13596                before_alive,
13597                "{kind:?}: death was published before recording"
13598            );
13599            assert!(before_history.entries.is_empty());
13600            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13601            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13602            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13603        }
13604    }
13605
13606    #[tokio::test]
13607    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13608        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13609        let path = dir.join("terminals.jsonl");
13610        std::fs::create_dir(&path).unwrap();
13611        let supervisor = Supervisor::new_for_test(
13612            Arc::new(Registry::default()),
13613            RestartPolicy::new(0, Duration::ZERO),
13614        )
13615        .with_terminal_journal(path, "health-budget-journal-failure".into());
13616        let runtime = supervisor.runtime_config();
13617        let spec = ModuleSpec {
13618            program: fake_aft_stub_path(),
13619            protocol: ModuleProtocol::None,
13620            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13621            ..windowed_crash_spec("health-budget-journal-failure")
13622        };
13623        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13624        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13625        health_restart_child(
13626            &spec,
13627            &runtime,
13628            &supervisor.registry,
13629            &supervisor.process_liveness,
13630            &snapshot,
13631            &mut child,
13632            SupervisorHealthStatus::Failing,
13633            None,
13634            1,
13635        )
13636        .await
13637        .unwrap();
13638        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13639        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13640        let history = runtime
13641            .terminal_ring
13642            .lock()
13643            .unwrap()
13644            .durable_history(&spec.module_id);
13645        assert!(history.journal_write_failures > 0);
13646        assert_eq!(history.entries.len(), 1);
13647        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13648    }
13649
13650    #[test]
13651    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13652        let supervisor = Supervisor::new_for_test(
13653            Arc::new(Registry::default()),
13654            RestartPolicy::new(0, Duration::ZERO),
13655        );
13656        let runtime = supervisor.runtime_config();
13657        let spec = ModuleSpec {
13658            program: fake_aft_stub_path(),
13659            protocol: ModuleProtocol::None,
13660            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13661            ..windowed_crash_spec("health-budget-recording-panic")
13662        };
13663        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13664        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13665            tokio::runtime::Builder::new_current_thread()
13666                .enable_all()
13667                .build()
13668                .unwrap()
13669                .block_on(async {
13670                    let mut child =
13671                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13672                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13673                    health_restart_child(
13674                        &spec,
13675                        &runtime,
13676                        &supervisor.registry,
13677                        &supervisor.process_liveness,
13678                        &snapshot,
13679                        &mut child,
13680                        SupervisorHealthStatus::Failing,
13681                        None,
13682                        1,
13683                    )
13684                    .await
13685                    .unwrap();
13686                });
13687        }));
13688        let panic = result.expect_err("recording must still unwind");
13689        assert_eq!(
13690            panic.downcast_ref::<String>().map(String::as_str),
13691            Some("spawn event sequence exhausted")
13692        );
13693        let state = lock_snapshot(&snapshot).unwrap();
13694        assert_eq!(state.state, ModuleState::Failed);
13695        assert!(!state.process_alive);
13696        assert!(state.last_exit.is_some());
13697    }
13698
13699    #[tokio::test]
13700    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13701        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13702        let path = dir.join("terminals.jsonl");
13703        std::fs::create_dir(&path).unwrap();
13704        let supervisor = Supervisor::new_for_test(
13705            Arc::new(Registry::default()),
13706            RestartPolicy::new(0, Duration::ZERO),
13707        )
13708        .with_terminal_journal(path, "budget-journal-failure".into());
13709        let runtime = supervisor.runtime_config();
13710        let spec = windowed_crash_spec("budget-journal-failure");
13711        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13712        assert!(matches!(
13713            on_child_exit(
13714                &spec,
13715                runtime.restart_policy,
13716                &supervisor.registry,
13717                &snapshot,
13718                &runtime.terminal_ring,
13719                &runtime.spawn_events,
13720                &runtime.child_roster,
13721                crash_exit_report(1),
13722                true,
13723            )
13724            .await,
13725            NextAction::Stop { .. }
13726        ));
13727        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13728        let history = runtime
13729            .terminal_ring
13730            .lock()
13731            .unwrap()
13732            .durable_history(&spec.module_id);
13733        assert!(history.journal_write_failures > 0);
13734        assert_eq!(history.entries.len(), 1);
13735        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13736    }
13737
13738    #[test]
13739    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13740        let supervisor = Supervisor::new_for_test(
13741            Arc::new(Registry::default()),
13742            RestartPolicy::new(0, Duration::ZERO),
13743        );
13744        let runtime = supervisor.runtime_config();
13745        let spec = windowed_crash_spec("budget-recording-panic");
13746        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13747        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13748        // Exhausting the event sequence makes emit_exited panic before the
13749        // terminal write, exercising publication on the recording unwind.
13750        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13751        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13752            tokio::runtime::Builder::new_current_thread()
13753                .enable_all()
13754                .build()
13755                .unwrap()
13756                .block_on(on_child_exit(
13757                    &spec,
13758                    runtime.restart_policy,
13759                    &supervisor.registry,
13760                    &snapshot,
13761                    &runtime.terminal_ring,
13762                    &runtime.spawn_events,
13763                    &runtime.child_roster,
13764                    crash_exit_report(1),
13765                    true,
13766                ))
13767        }));
13768        let panic = result.err().expect("recording must still unwind");
13769        assert_eq!(
13770            panic.downcast_ref::<String>().map(String::as_str),
13771            Some("spawn event sequence exhausted")
13772        );
13773        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13774    }
13775
13776    /// A stop the supervisor itself requests still stops a protocol-none
13777    /// module, even though the child answers the SIGTERM with exit 0.
13778    #[cfg(unix)]
13779    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13780    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13781        for disable in [false, true] {
13782            let label = if disable {
13783                "none-requested-disable"
13784            } else {
13785                "none-requested-stop"
13786            };
13787            let dir = cortexkit_test_support::ScratchDir::new(label);
13788            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13789            let supervisor = Supervisor::new_for_test(
13790                Arc::new(Registry::default()),
13791                RestartPolicy::new(3, Duration::ZERO),
13792            );
13793            let module = supervisor.spawn(spec).unwrap();
13794            wait_for_file(&ready).await;
13795
13796            if disable {
13797                module.set_enabled(false).await.unwrap();
13798            } else {
13799                module.stop().await.unwrap();
13800            }
13801            assert!(
13802                marker.exists(),
13803                "{label}: the child must have left through its SIGTERM handler with exit 0"
13804            );
13805
13806            // Long enough for a zero-backoff respawn to have happened if the
13807            // exit had been treated as a crash.
13808            sleep(Duration::from_millis(500)).await;
13809            let status = module.status().unwrap();
13810            let expected = if disable {
13811                ModuleState::Disabled
13812            } else {
13813                ModuleState::Stopped
13814            };
13815            assert_eq!(status.state, expected, "{label}");
13816            assert_eq!(
13817                status.spawn_generation, 1,
13818                "{label}: respawned after a requested stop"
13819            );
13820            let history = module.terminal_history();
13821            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13822            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13823            assert_ne!(
13824                history.entries[0].disposition,
13825                TerminalDisposition::Restarting,
13826                "{label}"
13827            );
13828        }
13829    }
13830
13831    /// A subc-wire module that exits 0 on its own is still a stop: the
13832    /// protocol-none rule must not reach it.
13833    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13834    async fn subc_wire_clean_exit_is_still_a_stop() {
13835        let supervisor = Supervisor::new_for_test(
13836            Arc::new(Registry::default()),
13837            RestartPolicy::new(3, Duration::ZERO),
13838        );
13839        let module = supervisor
13840            .spawn(ModuleSpec {
13841                module_id: "wire-clean-exit".to_string(),
13842                program: fake_aft_stub_path(),
13843                args: Vec::new(),
13844                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13845                reserved: false,
13846                reserved_prefixes: Vec::new(),
13847                protocol: ModuleProtocol::Subc,
13848                overlap: Default::default(),
13849                carry_containment: false,
13850            })
13851            .unwrap();
13852
13853        let deadline = Instant::now() + Duration::from_secs(10);
13854        while module.terminal_history().entries.is_empty() {
13855            assert!(Instant::now() < deadline, "module never exited");
13856            sleep(Duration::from_millis(10)).await;
13857        }
13858        // Long enough for a zero-backoff respawn to have happened.
13859        sleep(Duration::from_millis(500)).await;
13860        let status = module.status().unwrap();
13861        assert_eq!(status.state, ModuleState::Stopped);
13862        assert_eq!(status.spawn_generation, 1);
13863        let history = module.terminal_history();
13864        assert_eq!(history.entries.len(), 1, "{history:?}");
13865        assert_eq!(history.entries[0].exit_code, Some(0));
13866        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13867    }
13868
13869    #[cfg(unix)]
13870    #[tokio::test]
13871    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13872        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13873        let record = dir.join("live-children.json");
13874        let supervisor = Supervisor::new_for_test(
13875            Arc::new(Registry::default()),
13876            RestartPolicy::new(0, Duration::ZERO),
13877        );
13878        let mut runtime = supervisor.runtime_config();
13879        runtime.child_roster.record_to(record.clone());
13880        let gate = Arc::new(super::ReloadExitRecordGate::default());
13881        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13882        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13883        let spec = ModuleSpec {
13884            module_id: "reload-exit-roster".into(),
13885            program: fake_aft_stub_path(),
13886            args: Vec::new(),
13887            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13888            reserved: false,
13889            reserved_prefixes: Vec::new(),
13890            protocol: ModuleProtocol::Subc,
13891            overlap: Default::default(),
13892            carry_containment: false,
13893        };
13894        let mut child = None;
13895        let reload = super::finish_reload_child(
13896            &spec,
13897            &runtime,
13898            &supervisor.registry,
13899            &supervisor.process_liveness,
13900            &snapshot,
13901            &mut child,
13902        );
13903        tokio::pin!(reload);
13904        tokio::select! {
13905            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13906            _ = gate.reached.notified() => {}
13907        }
13908        assert!(runtime
13909            .terminal_ring
13910            .lock()
13911            .unwrap()
13912            .snapshot()
13913            .entries
13914            .is_empty());
13915        assert_eq!(
13916            crate::live_children::read_record(&record).unwrap().len(),
13917            1,
13918            "shutdown must still wait for the reaped child until its terminal record exists"
13919        );
13920        runtime.child_roster.close();
13921        gate.resume.notify_one();
13922        assert!(reload.await.is_err());
13923        assert!(crate::live_children::read_record(&record)
13924            .unwrap()
13925            .is_empty());
13926        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13927        assert_eq!(history.entries.len(), 1);
13928        assert_eq!(
13929            history.entries[0].disposition,
13930            TerminalDisposition::DaemonShutdown
13931        );
13932    }
13933
13934    /// Each restart-producing arm has its own state transition. Keeping their
13935    /// lifetime count assertions adjacent prevents a later new arm from silently
13936    /// spending budget without recording the historical restart.
13937    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13938    async fn every_restart_increment_path_advances_lifetime_count() {
13939        let supervisor = Supervisor::new_for_test(
13940            Arc::new(Registry::default()),
13941            RestartPolicy::new(1, Duration::ZERO),
13942        );
13943        let runtime = supervisor.runtime_config();
13944        let spec = ModuleSpec {
13945            module_id: "lifetime-increment-path".to_string(),
13946            program: PathBuf::from("/unused/lifetime-increment-path"),
13947            args: Vec::new(),
13948            env: Vec::new(),
13949            reserved: false,
13950            reserved_prefixes: Vec::new(),
13951            protocol: ModuleProtocol::Subc,
13952            overlap: Default::default(),
13953            carry_containment: false,
13954        };
13955
13956        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13957        assert!(matches!(
13958            on_child_exit(
13959                &spec,
13960                runtime.restart_policy,
13961                &supervisor.registry,
13962                &crash_snapshot,
13963                &runtime.terminal_ring,
13964                &runtime.spawn_events,
13965                &runtime.child_roster,
13966                ExitReport {
13967                    kind: ExitKind::Crash,
13968                    code: Some(1),
13969                    signal: None,
13970                    at_ms: 1,
13971                },
13972                true,
13973            )
13974            .await,
13975            NextAction::Restart { schedule: _ }
13976        ));
13977        let (crash_restarts, crash_lifetime) = {
13978            let state = lock_snapshot(&crash_snapshot).unwrap();
13979            (state.crash_restarts.len(), state.lifetime_restarts)
13980        };
13981        assert_eq!(crash_restarts, 1);
13982        assert_eq!(crash_lifetime, 1);
13983
13984        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13985        let mut health_child = None;
13986        assert!(matches!(
13987            health_restart_child(
13988                &spec,
13989                &runtime,
13990                &supervisor.registry,
13991                &supervisor.process_liveness,
13992                &health_snapshot,
13993                &mut health_child,
13994                SupervisorHealthStatus::Failing,
13995                None,
13996                2,
13997            )
13998            .await,
13999            Ok(())
14000        ));
14001        assert!(health_child.is_none());
14002        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
14003        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
14004        let (health_restarts, health_lifetime) = {
14005            let state = lock_snapshot(&health_snapshot).unwrap();
14006            (state.crash_restarts.len(), state.lifetime_restarts)
14007        };
14008        assert_eq!(health_restarts, 1);
14009        assert_eq!(health_lifetime, 1);
14010
14011        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14012        let mut reload_child = None;
14013        assert!(matches!(
14014            handle_reload_spawn_failure(
14015                &spec,
14016                &runtime,
14017                &supervisor.process_liveness,
14018                &reload_snapshot,
14019                &mut reload_child,
14020                "forced reload spawn failure".to_string(),
14021            )
14022            .await,
14023            Err(SuperviseError::ReloadFailed { .. })
14024        ));
14025        let (reload_restarts, reload_lifetime) = {
14026            let state = lock_snapshot(&reload_snapshot).unwrap();
14027            (state.crash_restarts.len(), state.lifetime_restarts)
14028        };
14029        assert_eq!(reload_restarts, 1);
14030        assert_eq!(reload_lifetime, 1);
14031    }
14032
14033    #[tokio::test]
14034    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14035        let supervisor = Supervisor::new_for_test(
14036            Arc::new(Registry::default()),
14037            RestartPolicy::new(3, Duration::ZERO),
14038        );
14039        let runtime = supervisor.runtime_config();
14040        let spec = ModuleSpec {
14041            module_id: "deliberately-severed".to_string(),
14042            program: PathBuf::from("/unused/deliberately-severed"),
14043            args: Vec::new(),
14044            env: Vec::new(),
14045            reserved: false,
14046            reserved_prefixes: Vec::new(),
14047            protocol: ModuleProtocol::Subc,
14048            overlap: Default::default(),
14049            carry_containment: false,
14050        };
14051        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14052        let process = ProcessIdentity {
14053            pid: 41,
14054            start_time: 101,
14055        };
14056        record_deliberate_severance(&snapshot, process).unwrap();
14057        let exit_report = apply_deliberate_severance_marker(
14058            &snapshot,
14059            Some(process),
14060            ExitReport {
14061                kind: ExitKind::Crash,
14062                code: Some(1),
14063                signal: None,
14064                at_ms: 1,
14065            },
14066        );
14067        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14068
14069        assert!(matches!(
14070            on_child_exit(
14071                &spec,
14072                runtime.restart_policy,
14073                &supervisor.registry,
14074                &snapshot,
14075                &runtime.terminal_ring,
14076                &runtime.spawn_events,
14077                &runtime.child_roster,
14078                exit_report,
14079                true,
14080            )
14081            .await,
14082            NextAction::Restart { schedule: _ }
14083        ));
14084        let state = lock_snapshot(&snapshot).unwrap();
14085        assert_eq!(state.lifetime_restarts, 1);
14086        assert_eq!(state.crash_restarts.len(), 0);
14087    }
14088
14089    #[tokio::test]
14090    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14091        let supervisor = Supervisor::new_for_test(
14092            Arc::new(Registry::default()),
14093            RestartPolicy::new(3, Duration::ZERO),
14094        );
14095        let runtime = supervisor.runtime_config();
14096        let spec = ModuleSpec {
14097            module_id: "genuine-crash".to_string(),
14098            program: PathBuf::from("/unused/genuine-crash"),
14099            args: Vec::new(),
14100            env: Vec::new(),
14101            reserved: false,
14102            reserved_prefixes: Vec::new(),
14103            protocol: ModuleProtocol::Subc,
14104            overlap: Default::default(),
14105            carry_containment: false,
14106        };
14107        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14108
14109        assert!(matches!(
14110            on_child_exit(
14111                &spec,
14112                runtime.restart_policy,
14113                &supervisor.registry,
14114                &snapshot,
14115                &runtime.terminal_ring,
14116                &runtime.spawn_events,
14117                &runtime.child_roster,
14118                ExitReport {
14119                    kind: ExitKind::Crash,
14120                    code: Some(1),
14121                    signal: None,
14122                    at_ms: 1,
14123                },
14124                true,
14125            )
14126            .await,
14127            NextAction::Restart { schedule: _ }
14128        ));
14129        let state = lock_snapshot(&snapshot).unwrap();
14130        assert_eq!(state.lifetime_restarts, 1);
14131        assert_eq!(state.crash_restarts.len(), 1);
14132    }
14133
14134    fn crash_exit_report(at_ms: u64) -> ExitReport {
14135        ExitReport {
14136            kind: ExitKind::Crash,
14137            code: Some(1),
14138            signal: None,
14139            at_ms,
14140        }
14141    }
14142
14143    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14144        ModuleSpec {
14145            module_id: module_id.to_string(),
14146            program: PathBuf::from("/unused").join(module_id),
14147            args: Vec::new(),
14148            env: Vec::new(),
14149            reserved: false,
14150            reserved_prefixes: Vec::new(),
14151            protocol: ModuleProtocol::Subc,
14152            overlap: Default::default(),
14153            carry_containment: false,
14154        }
14155    }
14156
14157    /// A real crash loop still stops. Three crashes with nothing aging out spend
14158    /// a budget of two and the third respawn is refused, and both surfaces an
14159    /// operator has -- the log line and the retained terminal record -- name the
14160    /// window rather than only the cap, because `max_restarts=2` alone is what
14161    /// this budget used to mean.
14162    #[tokio::test]
14163    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14164        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14165        let supervisor = Supervisor::new_for_test(
14166            Arc::new(Registry::default()),
14167            RestartPolicy::new(2, Duration::ZERO),
14168        );
14169        let runtime = supervisor.runtime_config();
14170        let spec = windowed_crash_spec("crash-loop-in-window");
14171        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14172
14173        for attempt in 1..=2 {
14174            assert!(
14175                matches!(
14176                    on_child_exit(
14177                        &spec,
14178                        runtime.restart_policy,
14179                        &supervisor.registry,
14180                        &snapshot,
14181                        &runtime.terminal_ring,
14182                        &runtime.spawn_events,
14183                        &runtime.child_roster,
14184                        crash_exit_report(attempt),
14185                        true,
14186                    )
14187                    .await,
14188                    NextAction::Restart { schedule: _ }
14189                ),
14190                "crash {attempt} is inside the budget and must respawn"
14191            );
14192        }
14193
14194        assert!(matches!(
14195            on_child_exit(
14196                &spec,
14197                runtime.restart_policy,
14198                &supervisor.registry,
14199                &snapshot,
14200                &runtime.terminal_ring,
14201                &runtime.spawn_events,
14202                &runtime.child_roster,
14203                crash_exit_report(3),
14204                true,
14205            )
14206            .await,
14207            NextAction::Stop { .. }
14208        ));
14209
14210        {
14211            let state = lock_snapshot(&snapshot).unwrap();
14212            assert_eq!(state.state, ModuleState::Failed);
14213            assert_eq!(state.crash_restarts.len(), 2);
14214            assert_eq!(state.lifetime_restarts, 2);
14215        }
14216
14217        let history = runtime
14218            .terminal_ring
14219            .lock()
14220            .expect("terminal ring is not poisoned")
14221            .snapshot();
14222        let last = history
14223            .entries
14224            .last()
14225            .expect("the refused crash is retained");
14226        assert_eq!(last.disposition, TerminalDisposition::Failed);
14227        assert_eq!(
14228            last.disposition_detail.as_deref(),
14229            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14230        );
14231
14232        let captured = crate::router::test_log::captured_logs(&logs);
14233        assert!(
14234            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14235            "the stop must be logged with its window: {captured}"
14236        );
14237    }
14238
14239    /// The rate, stated as a test: three crashes where the first has aged past
14240    /// the window are two crashes as far as the budget is concerned, so the
14241    /// third respawn is allowed and the ring holds only the two recent ones.
14242    ///
14243    /// This is the case a lifetime counter got wrong -- and the case the daemon
14244    /// now hits routinely, since a module exits non-zero every time its
14245    /// connection to the daemon drops.
14246    #[tokio::test]
14247    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14248        let supervisor = Supervisor::new_for_test(
14249            Arc::new(Registry::default()),
14250            RestartPolicy::new(2, Duration::ZERO),
14251        );
14252        let runtime = supervisor.runtime_config();
14253        let spec = windowed_crash_spec("crash-across-windows");
14254        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14255
14256        for attempt in 1..=2 {
14257            assert!(matches!(
14258                on_child_exit(
14259                    &spec,
14260                    runtime.restart_policy,
14261                    &supervisor.registry,
14262                    &snapshot,
14263                    &runtime.terminal_ring,
14264                    &runtime.spawn_events,
14265                    &runtime.child_roster,
14266                    crash_exit_report(attempt),
14267                    true,
14268                )
14269                .await,
14270                NextAction::Restart { schedule: _ }
14271            ));
14272        }
14273
14274        // The oldest crash moves out of the window; nothing else about the
14275        // module changes.
14276        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14277            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14278        })
14279        .unwrap();
14280
14281        assert!(
14282            matches!(
14283                on_child_exit(
14284                    &spec,
14285                    runtime.restart_policy,
14286                    &supervisor.registry,
14287                    &snapshot,
14288                    &runtime.terminal_ring,
14289                    &runtime.spawn_events,
14290                    &runtime.child_roster,
14291                    crash_exit_report(3),
14292                    true,
14293                )
14294                .await,
14295                NextAction::Restart { schedule: _ }
14296            ),
14297            "a crash older than the window must not hold a budget slot"
14298        );
14299
14300        let state = lock_snapshot(&snapshot).unwrap();
14301        assert_eq!(state.state, ModuleState::Restarting);
14302        assert_eq!(
14303            state.crash_restarts.len(),
14304            2,
14305            "the aged instant is dropped and the new one takes its place"
14306        );
14307        assert_eq!(
14308            state.lifetime_restarts, 3,
14309            "the ledger counts every restart, including the ones the window forgot"
14310        );
14311    }
14312
14313    /// An operator restart hands the budget back whole, and the ledger keeps
14314    /// counting. Those are different questions -- "how close is this module to
14315    /// being stopped" and "how many times has it been replaced" -- and the
14316    /// operator action answers only the first.
14317    #[tokio::test]
14318    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14319        let supervisor = Supervisor::new_for_test(
14320            Arc::new(Registry::default()),
14321            RestartPolicy::new(2, Duration::ZERO),
14322        );
14323        let runtime = supervisor.runtime_config();
14324        let spec = windowed_crash_spec("operator-cleared-budget");
14325        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14326
14327        for attempt in 1..=2 {
14328            assert!(matches!(
14329                on_child_exit(
14330                    &spec,
14331                    runtime.restart_policy,
14332                    &supervisor.registry,
14333                    &snapshot,
14334                    &runtime.terminal_ring,
14335                    &runtime.spawn_events,
14336                    &runtime.child_roster,
14337                    crash_exit_report(attempt),
14338                    true,
14339                )
14340                .await,
14341                NextAction::Restart { schedule: _ }
14342            ));
14343        }
14344
14345        reset_restart_count(&snapshot, &spec.module_id).unwrap();
14346        {
14347            let state = lock_snapshot(&snapshot).unwrap();
14348            assert!(
14349                state.crash_restarts.is_empty(),
14350                "an operator restart returns the full budget"
14351            );
14352            assert_eq!(
14353                state.lifetime_restarts, 2,
14354                "clearing the budget must not unmake the crashes"
14355            );
14356        }
14357
14358        assert!(
14359            matches!(
14360                on_child_exit(
14361                    &spec,
14362                    runtime.restart_policy,
14363                    &supervisor.registry,
14364                    &snapshot,
14365                    &runtime.terminal_ring,
14366                    &runtime.spawn_events,
14367                    &runtime.child_roster,
14368                    crash_exit_report(3),
14369                    true,
14370                )
14371                .await,
14372                NextAction::Restart { schedule: _ }
14373            ),
14374            "the cleared budget must be spendable again"
14375        );
14376        let state = lock_snapshot(&snapshot).unwrap();
14377        assert_eq!(state.crash_restarts.len(), 1);
14378        assert_eq!(state.lifetime_restarts, 3);
14379    }
14380
14381    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14382    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14383        let severed = ProcessIdentity {
14384            pid: 41,
14385            start_time: 101,
14386        };
14387        let successor = ProcessIdentity {
14388            pid: 41,
14389            start_time: 202,
14390        };
14391        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14392        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14393            state.pid = Some(successor.pid);
14394            state.process_start_time = Some(successor.start_time);
14395        })
14396        .unwrap();
14397        assert!(!module.record_deliberate_severance(severed).unwrap());
14398
14399        let exit_report = apply_deliberate_severance_marker(
14400            &module.inner.snapshot,
14401            Some(successor),
14402            ExitReport {
14403                kind: ExitKind::Crash,
14404                code: Some(1),
14405                signal: None,
14406                at_ms: 1,
14407            },
14408        );
14409
14410        assert_eq!(exit_report.kind, ExitKind::Crash);
14411    }
14412
14413    #[tokio::test]
14414    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14415        let registry = Registry::default();
14416        let supervisor = Supervisor::new_for_test(
14417            Arc::new(Registry::default()),
14418            RestartPolicy::new(3, Duration::ZERO),
14419        );
14420        let runtime = supervisor.runtime_config();
14421        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14422        let spec = ModuleSpec {
14423            module_id: "drain-deliberate-severance".to_string(),
14424            program: fake_aft_stub_path(),
14425            args: Vec::new(),
14426            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14427            reserved: false,
14428            reserved_prefixes: Vec::new(),
14429            protocol: ModuleProtocol::Subc,
14430            overlap: Default::default(),
14431            carry_containment: false,
14432        };
14433        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14434        let process = ProcessIdentity {
14435            pid: 41,
14436            start_time: 101,
14437        };
14438        child.process_identity = Some(process);
14439        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14440            state.pid = Some(process.pid);
14441            state.process_start_time = Some(process.start_time);
14442        })
14443        .unwrap();
14444        record_deliberate_severance(&snapshot, process).unwrap();
14445
14446        drain_child_to_state(
14447            &spec.module_id,
14448            spec.protocol,
14449            // The child exits on its own; no signal may change the exit this
14450            // test classifies.
14451            StopNotice::SentOverConnection,
14452            &registry,
14453            None,
14454            &snapshot,
14455            &runtime.terminal_ring,
14456            &runtime.spawn_events,
14457            child,
14458            Duration::from_secs(1),
14459            ModuleState::Stopped,
14460            Some(false),
14461            &super::FailureContext::operator(),
14462        )
14463        .await
14464        .unwrap();
14465
14466        let state = lock_snapshot(&snapshot).unwrap();
14467        assert_eq!(
14468            state.last_exit.as_ref().map(|exit| exit.kind),
14469            Some(ExitKind::DeliberateSeverance)
14470        );
14471        assert_eq!(state.lifetime_restarts, 1);
14472        assert_eq!(state.crash_restarts.len(), 0);
14473        drop(state);
14474        let history = runtime.terminal_ring.lock().unwrap().snapshot();
14475        assert_eq!(
14476            history.entries[0].exit_kind,
14477            subc_control::TerminalExitKind::DeliberateSeverance
14478        );
14479    }
14480
14481    #[tokio::test]
14482    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14483        let registry = Registry::default();
14484        let supervisor = Supervisor::new_for_test(
14485            Arc::new(Registry::default()),
14486            RestartPolicy::new(3, Duration::ZERO),
14487        );
14488        let runtime = supervisor.runtime_config();
14489        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14490        let spec = ModuleSpec {
14491            module_id: "ordinary-drain".to_string(),
14492            program: fake_aft_stub_path(),
14493            args: Vec::new(),
14494            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14495            reserved: false,
14496            reserved_prefixes: Vec::new(),
14497            protocol: ModuleProtocol::Subc,
14498            overlap: Default::default(),
14499            carry_containment: false,
14500        };
14501        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14502
14503        drain_child_to_state(
14504            &spec.module_id,
14505            spec.protocol,
14506            // The child exits on its own; no signal may change the exit this
14507            // test classifies.
14508            StopNotice::SentOverConnection,
14509            &registry,
14510            None,
14511            &snapshot,
14512            &runtime.terminal_ring,
14513            &runtime.spawn_events,
14514            child,
14515            Duration::from_secs(1),
14516            ModuleState::Stopped,
14517            Some(false),
14518            &super::FailureContext::operator(),
14519        )
14520        .await
14521        .unwrap();
14522
14523        let state = lock_snapshot(&snapshot).unwrap();
14524        assert_eq!(
14525            state.last_exit.as_ref().map(|exit| exit.kind),
14526            Some(ExitKind::Crash)
14527        );
14528        assert_eq!(state.lifetime_restarts, 0);
14529        assert_eq!(state.crash_restarts.len(), 0);
14530    }
14531
14532    #[test]
14533    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14534        // The server's generic fatal-routing branch only knows that the
14535        // connection failed; it does not know that the daemon deliberately
14536        // initiated a process-killing severance. Keep this seam explicit so a
14537        // future connection error path cannot silently reintroduce the stale
14538        // exemption that mislabels a later genuine crash.
14539        assert!(!include_str!("server.rs")
14540            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14541    }
14542
14543    /// The `route.closed` `drained` value must be the quiescence wait's own
14544    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
14545    /// measurement at all and `false` is the one honest constant. This is the exact
14546    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
14547    /// on every return path, including the one that used to return early via `?`
14548    /// with `route.closing` already sent and no `route.closed` ever following.
14549    #[test]
14550    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14551        assert!(drained_after_quiescence_wait(&Ok(true)));
14552        assert!(!drained_after_quiescence_wait(&Ok(false)));
14553        assert!(!drained_after_quiescence_wait(&Err(
14554            SuperviseError::StatePoisoned { module_id: None }
14555        )));
14556    }
14557
14558    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
14559    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
14560    /// already reaped out-of-band) still leaves a terminal record rather than none
14561    /// at all. Triggering the real `wait()` I/O error from an integration test would
14562    /// need a genuine already-reaped-child race, which is OS-specific and not
14563    /// something this suite attempts elsewhere; this test instead verifies the
14564    /// record produced for that arm end-to-end through the real `TerminalRing`, and
14565    /// the call site itself is verified by inspection to sit in that exact arm.
14566    #[test]
14567    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14568        let ring = Arc::new(Mutex::new(TerminalRing::new(
14569            TerminalRingConfig::default(),
14570            0,
14571        )));
14572        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14573
14574        let snapshot = ring.lock().unwrap().snapshot();
14575        assert_eq!(snapshot.entries.len(), 1);
14576        let entry = &snapshot.entries[0];
14577        assert_eq!(entry.exit_code, None);
14578        assert_eq!(entry.exit_signal, None);
14579        assert_eq!(entry.disposition, TerminalDisposition::Failed);
14580    }
14581
14582    #[test]
14583    fn wait_error_exit_path_preserves_spawn_event_density() {
14584        let feed = super::SpawnEventFeed::default();
14585        feed.configure_incarnation("wait-error-density".to_string());
14586        feed.emit_spawned("wait-error", 41, 1);
14587        let ring = Arc::new(Mutex::new(TerminalRing::new(
14588            TerminalRingConfig::default(),
14589            0,
14590        )));
14591
14592        record_wait_error_terminal("wait-error", &ring, &feed);
14593        feed.emit_spawned("after-wait-error", 42, 2);
14594
14595        let state = feed.0.lock().unwrap();
14596        let sequences = state
14597            .events
14598            .iter()
14599            .map(|event| event.cursor.seq)
14600            .collect::<Vec<_>>();
14601        assert_eq!(sequences, vec![1, 2, 3]);
14602        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14603        assert_eq!(state.events[1].exit_code, None);
14604        assert_eq!(state.events[1].exit_signal, None);
14605    }
14606
14607    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
14608    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
14609    /// not a clean exit it never actually observed.
14610    #[test]
14611    fn wait_error_exit_report_is_classified_as_a_crash() {
14612        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14613    }
14614}
14615
14616#[cfg(all(test, unix))]
14617mod restart_ordering_tests {
14618    use super::*;
14619    use std::io::Write;
14620    use tokio::io::AsyncReadExt;
14621
14622    #[test]
14623    #[ignore = "child process fixture"]
14624    fn serving_child() {
14625        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14626            return;
14627        };
14628        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14629        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14630        for stream in listener.incoming() {
14631            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14632        }
14633    }
14634
14635    async fn restart_before_publication(requests: usize) {
14636        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14637        let address_file = home.path().join("address");
14638        let spec = ModuleSpec {
14639            module_id: "restart-serving-child".into(),
14640            program: std::env::current_exe().unwrap(),
14641            args: vec![
14642                "--exact".into(),
14643                "supervise::restart_ordering_tests::serving_child".into(),
14644                "--ignored".into(),
14645                "--nocapture".into(),
14646            ],
14647            env: vec![(
14648                "RESTART_TEST_ADDRESS_FILE".into(),
14649                address_file.display().to_string(),
14650            )],
14651            reserved: false,
14652            reserved_prefixes: vec![],
14653            protocol: ModuleProtocol::None,
14654            overlap: Default::default(),
14655            carry_containment: false,
14656        };
14657        let mut supervisor =
14658            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14659        let (spawned_tx, spawned_rx) = oneshot::channel();
14660        let spawned_tx = Mutex::new(Some(spawned_tx));
14661        let (release_tx, release_rx) = std::sync::mpsc::channel();
14662        let release_rx = Mutex::new(release_rx);
14663        // Hold publication, not the child: it must answer a real request before
14664        // the restart enters the queue, even though status still has generation 0.
14665        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14666            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14667            release_rx
14668                .lock()
14669                .unwrap()
14670                .recv_timeout(Duration::from_secs(20))
14671                .unwrap();
14672        })));
14673        let registered = supervisor
14674            .register_configured_with_health(
14675                spec.clone(),
14676                true,
14677                HealthConfig::default(),
14678                None,
14679                RestartPolicy::new(3, Duration::ZERO),
14680                DEFAULT_REGISTER_TIMEOUT,
14681            )
14682            .unwrap();
14683        let module = registered.module.clone();
14684        let startup = tokio::task::spawn_blocking(move || {
14685            supervisor.start_registered_configured(registered, None, true)
14686        });
14687        spawned_rx.await.unwrap();
14688        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14689            let address = loop {
14690                if let Ok(address) = std::fs::read_to_string(&address_file) {
14691                    if !address.is_empty() {
14692                        break address;
14693                    }
14694                }
14695                tokio::time::sleep(Duration::from_millis(1)).await;
14696            };
14697            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14698            let mut reply = String::new();
14699            stream.read_to_string(&mut reply).await.unwrap();
14700            reply.trim().parse::<u32>().unwrap()
14701        })
14702        .await
14703        .expect("child must serve before generation publication");
14704        assert_eq!(
14705            lock_snapshot(&module.inner.snapshot)
14706                .unwrap()
14707                .spawn_generation,
14708            0
14709        );
14710        let mut replies = Vec::new();
14711        for _ in 0..requests {
14712            let mut restart = Box::pin(module.restart(Some(0)));
14713            // Poll the public API through queue admission, leaving only its ACK
14714            // pending. The monitor cannot consume the command until released.
14715            std::future::poll_fn(|cx| {
14716                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14717                std::task::Poll::Ready(())
14718            })
14719            .await;
14720            replies.push(restart);
14721        }
14722        release_tx.send(()).unwrap();
14723        startup.await.unwrap().unwrap();
14724        for reply in replies {
14725            reply.await.unwrap();
14726        }
14727        // Restart schedules its replacement after completing the handler. Wait
14728        // for that separate actor turn before placing the command barrier.
14729        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14730            loop {
14731                if lock_snapshot(&module.inner.snapshot)
14732                    .unwrap()
14733                    .spawn_generation
14734                    > 1
14735                {
14736                    break;
14737                }
14738                tokio::time::sleep(Duration::from_millis(1)).await;
14739            }
14740        })
14741        .await;
14742        // Commands are handled serially. The update-spec reply proves every
14743        // restart handler has finished before we inspect the snapshot; restart's
14744        // own ACK only acknowledges initiation, not replacement completion.
14745        module.update_spec_for_test(spec).await.unwrap();
14746        let (generation, pid) = {
14747            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14748            (state.spawn_generation, state.pid)
14749        };
14750        module.stop().await.unwrap();
14751        replacement.expect("restart must publish a replacement for the already-serving child");
14752        assert_ne!(
14753            pid,
14754            Some(old_pid),
14755            "a serving child predating the restart must be replaced"
14756        );
14757        assert_eq!(generation, 2, "queued restarts must share one replacement");
14758    }
14759
14760    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14761    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14762        restart_before_publication(1).await;
14763    }
14764
14765    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14766    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14767        restart_before_publication(2).await;
14768    }
14769
14770    #[tokio::test]
14771    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14772        let supervisor =
14773            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14774        let registered = supervisor
14775            .register_configured_with_health(
14776                ModuleSpec {
14777                    module_id: "queued-before-replacement".into(),
14778                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14779                    // macOS a module whose running image is not the configured
14780                    // program is refused once its exec is confirmed (see
14781                    // `record_write_stall_tests::sleeping_spec`).
14782                    program: "/bin/sleep".into(),
14783                    args: vec!["600".into()],
14784                    env: vec![],
14785                    reserved: false,
14786                    reserved_prefixes: vec![],
14787                    protocol: ModuleProtocol::None,
14788                    overlap: Default::default(),
14789                    carry_containment: false,
14790                },
14791                true,
14792                HealthConfig::default(),
14793                None,
14794                RestartPolicy::default(),
14795                DEFAULT_REGISTER_TIMEOUT,
14796            )
14797            .unwrap();
14798        let RegisteredConfiguredModule {
14799            mut spec,
14800            mut runtime,
14801            module,
14802            mut commands,
14803        } = registered;
14804        let snapshot = &module.inner.snapshot;
14805        let mut restart = Box::pin(module.restart(Some(0)));
14806        std::future::poll_fn(|cx| {
14807            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14808            std::task::Poll::Ready(())
14809        })
14810        .await;
14811        // Drive the actor explicitly so a replacement is definitely spawned
14812        // after receipt, but before this queued command is dequeued.
14813        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14814        let replacement_pid = child.as_ref().unwrap().id();
14815        let command = commands.recv().await.unwrap();
14816        assert!(
14817            handle_supervisor_command(
14818                command,
14819                &mut spec,
14820                &mut runtime,
14821                &supervisor.registry,
14822                &supervisor.process_liveness,
14823                snapshot,
14824                &mut child,
14825                &mut commands,
14826                &mut VecDeque::new(),
14827            )
14828            .await
14829        );
14830        restart.await.unwrap();
14831        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14832        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14833        if let Some(mut child) = child {
14834            child.start_kill().unwrap();
14835            child.wait().await.unwrap();
14836        }
14837        assert_eq!(
14838            kept_pid, replacement_pid,
14839            "a newer replacement must survive"
14840        );
14841        assert!(
14842            !pending,
14843            "a satisfied restart must not schedule another spawn"
14844        );
14845    }
14846}
14847
14848#[cfg(all(test, unix))]
14849mod health_event_tests {
14850    use super::*;
14851    use cortexkit_test_support::ScratchDir;
14852    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14853
14854    struct Actor {
14855        module: SupervisedModule,
14856        registry: Arc<Registry>,
14857        forwarding: Arc<ForwardingTable>,
14858        spec: ModuleSpec,
14859        health: HealthConfig,
14860        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14861        roster: ChildRoster,
14862        _home: ScratchDir,
14863    }
14864
14865    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14866        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14867        // for an observed condition, rather than letting paused time auto-advance
14868        // deadlines or assuming a fixed number of yields finishes the work.
14869        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14870        let tick = Instant::now();
14871        loop {
14872            if let Some(value) = observe() {
14873                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14874                return value;
14875            }
14876            assert!(
14877                std::time::Instant::now() < deadline,
14878                "timed out waiting for {reason}"
14879            );
14880            tokio::task::yield_now().await;
14881        }
14882    }
14883
14884    impl Actor {
14885        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14886            Self::start_with_policy(
14887                protocol,
14888                cadence,
14889                DEFAULT_REGISTER_TIMEOUT,
14890                RestartPolicy::new(3, Duration::ZERO),
14891            )
14892            .await
14893        }
14894
14895        async fn start_with_policy(
14896            protocol: ModuleProtocol,
14897            cadence: Duration,
14898            register_timeout: Duration,
14899            policy: RestartPolicy,
14900        ) -> Self {
14901            let home = ScratchDir::new("health-events");
14902            let registry = Arc::new(Registry::default());
14903            let forwarding = Arc::new(ForwardingTable::default());
14904            let health = HealthConfig {
14905                cadence,
14906                deadline: Duration::from_secs(3600),
14907                ..HealthConfig::default()
14908            };
14909            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14910                .with_forwarding(forwarding.clone())
14911                .with_health_config(health.clone());
14912            let spec = ModuleSpec {
14913                module_id: "event-health-child".into(),
14914                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14915                // the privacy trampoline refuses a module whose running image
14916                // is not the configured program, and `start` below waits for
14917                // the process to be confirmed (`reported_pid`). The shell form
14918                // passed only while the image check ran before the shell had
14919                // replaced itself with `sleep` (see
14920                // `record_write_stall_tests::sleeping_spec`).
14921                program: PathBuf::from("/bin/sleep"),
14922                // The process stays alive but has no real bus peer. Registrations
14923                // below exercise the same registry writes as accepted HELLOs.
14924                args: vec!["600".into()],
14925                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14926                    .into_iter()
14927                    .map(|key| (key.into(), home.path().display().to_string()))
14928                    .collect(),
14929                reserved: false,
14930                reserved_prefixes: vec![],
14931                protocol,
14932                overlap: Default::default(),
14933                carry_containment: false,
14934            };
14935            let module = supervisor
14936                .supervise_configured_requested(
14937                    spec.clone(),
14938                    true,
14939                    health.clone(),
14940                    Some(0),
14941                    policy,
14942                    register_timeout,
14943                    None,
14944                )
14945                .unwrap();
14946            let (_, rx) = mpsc::channel(8);
14947            let actor = Self {
14948                module,
14949                registry,
14950                forwarding,
14951                spec,
14952                health,
14953                rx,
14954                roster: supervisor.child_roster.clone(),
14955                _home: home,
14956            };
14957            actor
14958                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14959                .await;
14960            actor
14961        }
14962
14963        async fn expire_registration_wait(&self, remaining: Duration) {
14964            tokio::time::advance(remaining).await;
14965            wait_for("registration timeout TERM grace", || {
14966                lock_snapshot(&self.module.inner.snapshot)
14967                    .unwrap()
14968                    .actor_select
14969                    .is_none()
14970                    .then_some(())
14971            })
14972            .await;
14973            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14974        }
14975
14976        fn turns(&self) -> u64 {
14977            lock_snapshot(&self.module.inner.snapshot)
14978                .unwrap()
14979                .actor_turns
14980        }
14981
14982        async fn wait_for_select(
14983            &self,
14984            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14985        ) -> ActorSelectCheckpoint {
14986            wait_for("actor select checkpoint", || {
14987                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14988                state.actor_select.filter(|checkpoint| {
14989                    checkpoint.generation == state.spawn_generation
14990                        && state.state == ModuleState::Running
14991                        && state.process_alive
14992                        && state.reported_pid().is_some()
14993                        && expected(checkpoint)
14994                })
14995            })
14996            .await
14997        }
14998
14999        fn hello(&mut self, connection: u64, health: bool) {
15000            let manifest =
15001                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
15002                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
15003                    .build();
15004            let connection = ConnectionId::new(connection);
15005            let (tx, rx) = mpsc::channel(8);
15006            self.rx = rx;
15007            self.forwarding
15008                .register_module_connection(
15009                    connection,
15010                    self.spec.module_id.clone(),
15011                    subc_protocol::PROTOCOL_VERSION,
15012                    Concurrency::ModuleManaged,
15013                    FrameSink::new(tx),
15014                )
15015                .unwrap();
15016            self.registry
15017                .register_with_control_ops(
15018                    manifest,
15019                    subc_protocol::PROTOCOL_VERSION,
15020                    connection,
15021                    if health {
15022                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15023                    } else {
15024                        vec![]
15025                    },
15026                )
15027                .unwrap();
15028        }
15029
15030        async fn probe(&mut self) -> crate::router::OutboundFrame {
15031            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15032                Ok(frame) => Some(frame),
15033                Err(mpsc::error::TryRecvError::Empty) => None,
15034                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15035            })
15036            .await;
15037            let body: Value = serde_json::from_slice(&frame.body).unwrap();
15038            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15039            frame
15040        }
15041
15042        fn no_probe(&mut self) {
15043            assert!(matches!(
15044                self.rx.try_recv(),
15045                Err(mpsc::error::TryRecvError::Empty)
15046            ));
15047        }
15048
15049        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15050            self.forwarding
15051                .complete_module_control_rpc(
15052                    ConnectionId::new(connection),
15053                    frame.header.corr,
15054                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15055                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15056                        status: HealthStatus::Ok,
15057                        detail: None,
15058                        metrics: None,
15059                    }),
15060                )
15061                .unwrap();
15062        }
15063    }
15064
15065    #[tokio::test(start_paused = true)]
15066    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15067        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15068        let initial = actor.module.status().unwrap();
15069        tokio::time::advance(Duration::from_secs(119)).await;
15070        tokio::task::yield_now().await;
15071        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15072        assert!(actor.module.terminal_history().entries.is_empty());
15073        actor.expire_registration_wait(Duration::from_secs(1)).await;
15074        actor
15075            .wait_for_select(|at| at.generation > initial.spawn_generation)
15076            .await;
15077        let status = actor.module.status().unwrap();
15078        assert_ne!(status.pid, initial.pid);
15079        assert!(!cortexkit_test_support::process_alive(
15080            initial.pid.unwrap() as i32
15081        ));
15082        assert_eq!(status.restart_count, 1);
15083        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15084        let entries = actor.module.terminal_history().entries;
15085        assert_eq!(entries.len(), 1);
15086        assert_eq!(
15087            entries[0].disposition_detail.as_deref(),
15088            Some("registration_timeout")
15089        );
15090        actor.module.stop().await.unwrap();
15091    }
15092
15093    #[tokio::test(start_paused = true)]
15094    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15095        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15096        let pid = actor.module.status().unwrap().pid;
15097        tokio::time::advance(Duration::from_secs(119)).await;
15098        actor.hello(1, false);
15099        let registered = actor
15100            .wait_for_select(|at| at.registered_connection.is_some())
15101            .await;
15102        assert_eq!(
15103            registered.wake_after, None,
15104            "HELLO must cancel the deadline"
15105        );
15106        tokio::time::advance(Duration::from_secs(300)).await;
15107        tokio::task::yield_now().await;
15108        assert_eq!(actor.module.status().unwrap().pid, pid);
15109        assert!(actor.module.terminal_history().entries.is_empty());
15110        // A disconnect after the first HELLO must not re-arm a startup timer.
15111        actor
15112            .registry
15113            .deregister_connection(ConnectionId::new(1))
15114            .unwrap();
15115        actor
15116            .wait_for_select(|at| at.registered_connection.is_none())
15117            .await;
15118        tokio::time::advance(Duration::from_secs(300)).await;
15119        tokio::task::yield_now().await;
15120        assert_eq!(actor.module.status().unwrap().pid, pid);
15121        assert!(actor.module.terminal_history().entries.is_empty());
15122        actor.module.stop().await.unwrap();
15123
15124        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15125        let pid = transient.module.status().unwrap().pid;
15126        tokio::time::advance(Duration::from_secs(119)).await;
15127        let turns = transient.turns();
15128        // Both writes happen before the supervisor can refresh the registry.
15129        // An accepted HELLO must remain evidence after its connection is gone.
15130        transient.hello(1, false);
15131        transient
15132            .registry
15133            .deregister_connection(ConnectionId::new(1))
15134            .unwrap();
15135        let parked = transient.wait_for_select(|at| at.turn > turns).await;
15136        assert_eq!(
15137            parked.wake_after, None,
15138            "transient HELLO must cancel the deadline"
15139        );
15140        tokio::time::advance(Duration::from_secs(300)).await;
15141        tokio::task::yield_now().await;
15142        assert_eq!(transient.module.status().unwrap().pid, pid);
15143        assert!(transient.module.terminal_history().entries.is_empty());
15144        transient.module.stop().await.unwrap();
15145    }
15146
15147    #[tokio::test(start_paused = true)]
15148    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15149        let actor = Actor::start_with_policy(
15150            ModuleProtocol::Subc,
15151            Duration::from_secs(30),
15152            Duration::from_secs(10),
15153            RestartPolicy::new(1, Duration::ZERO),
15154        )
15155        .await;
15156        let generation = actor.module.status().unwrap().spawn_generation;
15157        actor
15158            .expire_registration_wait(Duration::from_secs(10))
15159            .await;
15160        actor.wait_for_select(|at| at.generation > generation).await;
15161        actor
15162            .expire_registration_wait(Duration::from_secs(10))
15163            .await;
15164        let retry = wait_for("failed retry schedule", || {
15165            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15166            (state.state == ModuleState::Failed)
15167                .then_some(state.failed_retry.pending)
15168                .flatten()
15169        })
15170        .await;
15171        assert_eq!(retry.delay, Duration::from_secs(600));
15172        assert_eq!(retry.attempt, 1);
15173        let entries = actor.module.terminal_history().entries;
15174        assert_eq!(entries.len(), 2);
15175        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15176        assert!(entries[1]
15177            .disposition_detail
15178            .as_deref()
15179            .unwrap()
15180            .contains("registration_timeout"));
15181        tokio::time::advance(Duration::from_secs(599)).await;
15182        tokio::task::yield_now().await;
15183        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15184        tokio::time::advance(Duration::from_secs(1)).await;
15185        actor
15186            .wait_for_select(|at| at.generation > generation + 1)
15187            .await;
15188        // The slow retry is another ordinary spawn, with its own HELLO budget.
15189        actor
15190            .expire_registration_wait(Duration::from_secs(10))
15191            .await;
15192        wait_for("slow retry registration timeout", || {
15193            (actor.module.terminal_history().entries.len() == 3).then_some(())
15194        })
15195        .await;
15196        actor.module.stop().await.unwrap();
15197    }
15198
15199    #[tokio::test(start_paused = true)]
15200    async fn registration_timeout_honours_ten_second_override() {
15201        let actor = Actor::start_with_policy(
15202            ModuleProtocol::Subc,
15203            Duration::from_secs(30),
15204            Duration::from_secs(10),
15205            RestartPolicy::new(3, Duration::ZERO),
15206        )
15207        .await;
15208        let generation = actor.module.status().unwrap().spawn_generation;
15209        tokio::time::advance(Duration::from_secs(9)).await;
15210        tokio::task::yield_now().await;
15211        assert!(actor.module.terminal_history().entries.is_empty());
15212        actor.expire_registration_wait(Duration::from_secs(1)).await;
15213        actor.wait_for_select(|at| at.generation > generation).await;
15214        assert_eq!(actor.module.terminal_history().entries.len(), 1);
15215        actor.module.stop().await.unwrap();
15216    }
15217
15218    #[tokio::test]
15219    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15220        use std::os::unix::process::{CommandExt, ExitStatusExt};
15221
15222        struct Helper(std::process::Child);
15223        impl Drop for Helper {
15224            fn drop(&mut self) {
15225                // This handle still owns the helper until wait returns. Never
15226                // clean up its old process group after the leader is reaped.
15227                if matches!(self.0.try_wait(), Ok(None)) {
15228                    let _ = self.0.kill();
15229                    let _ = self.0.wait();
15230                }
15231            }
15232        }
15233
15234        let home = ScratchDir::new("registration-timeout-helper");
15235        let supervisor = Supervisor::new_for_test(
15236            Arc::new(Registry::default()),
15237            RestartPolicy::new(0, Duration::ZERO),
15238        );
15239        let spec = ModuleSpec {
15240            module_id: "registration-timeout-group".into(),
15241            program: "/bin/sleep".into(),
15242            args: vec!["120".into()],
15243            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15244                .into_iter()
15245                .map(|key| (key.into(), home.path().display().to_string()))
15246                .collect(),
15247            reserved: false,
15248            reserved_prefixes: vec![],
15249            protocol: ModuleProtocol::Subc,
15250            overlap: Default::default(),
15251            carry_containment: false,
15252        };
15253        let module = supervisor
15254            .supervise_configured_requested(
15255                spec,
15256                true,
15257                HealthConfig::default(),
15258                None,
15259                RestartPolicy::new(0, Duration::ZERO),
15260                Duration::from_secs(10),
15261                None,
15262            )
15263            .unwrap();
15264        let leader = timeout(Duration::from_secs(5), async {
15265            loop {
15266                if let Some(pid) = module.status().unwrap().pid {
15267                    break pid;
15268                }
15269                sleep(Duration::from_millis(5)).await;
15270            }
15271        })
15272        .await
15273        .expect("supervised group leader must be reported before the deadline");
15274        assert!(leader > 1);
15275        let ready = home.join("helper.ready");
15276        // Own the helper separately so even a broken group kill can be cleaned
15277        // up by its own unreaped child handle, without signalling a stale pgid.
15278        let mut helper = Helper(
15279            std::process::Command::new("/bin/sh")
15280                .args([
15281                    "-c",
15282                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15283                    "helper",
15284                ])
15285                .arg(&ready)
15286                .process_group(i32::try_from(leader).unwrap())
15287                .stdout(std::process::Stdio::null())
15288                .stderr(std::process::Stdio::null())
15289                .spawn()
15290                .unwrap(),
15291        );
15292        timeout(Duration::from_secs(5), async {
15293            while !ready.exists() {
15294                assert!(helper.0.try_wait().unwrap().is_none());
15295                sleep(Duration::from_millis(5)).await;
15296            }
15297        })
15298        .await
15299        .expect("helper must install TERM ignore before the registration deadline");
15300        let helper_pid =
15301            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15302        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15303        sleep(Duration::from_millis(20)).await;
15304        assert!(
15305            helper.0.try_wait().unwrap().is_none(),
15306            "helper must ignore TERM"
15307        );
15308
15309        timeout(Duration::from_secs(20), async {
15310            while module.state().unwrap() != ModuleState::Failed {
15311                sleep(Duration::from_millis(5)).await;
15312            }
15313        })
15314        .await
15315        .expect("registration timeout must stop and reap the group leader");
15316        let helper_exit = timeout(Duration::from_secs(3), async {
15317            loop {
15318                if let Some(status) = helper.0.try_wait().unwrap() {
15319                    break status;
15320                }
15321                sleep(Duration::from_millis(5)).await;
15322            }
15323        })
15324        .await
15325        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15326        assert_eq!(helper_exit.signal(), Some(9));
15327        assert!(!cortexkit_test_support::process_alive(
15328            i32::try_from(leader).unwrap()
15329        ));
15330        assert_eq!(
15331            module.status().unwrap().last_exit.unwrap().kind,
15332            ExitKind::Crash
15333        );
15334        assert!(module.terminal_history().entries[0]
15335            .disposition_detail
15336            .as_deref()
15337            .unwrap()
15338            .contains("registration_timeout"));
15339        module.stop().await.unwrap();
15340    }
15341
15342    #[tokio::test(start_paused = true)]
15343    async fn registration_timeout_does_not_count_an_earlier_exit() {
15344        let actor = Actor::start_with_policy(
15345            ModuleProtocol::Subc,
15346            Duration::from_secs(30),
15347            DEFAULT_REGISTER_TIMEOUT,
15348            RestartPolicy::new(0, Duration::ZERO),
15349        )
15350        .await;
15351        let pid = actor.module.status().unwrap().pid.unwrap();
15352        rustix::process::kill_process(
15353            rustix::process::Pid::from_raw(pid as i32).unwrap(),
15354            rustix::process::Signal::TERM,
15355        )
15356        .unwrap();
15357        wait_for("early exit", || {
15358            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15359        })
15360        .await;
15361        tokio::time::advance(Duration::from_secs(121)).await;
15362        tokio::task::yield_now().await;
15363        let entries = actor.module.terminal_history().entries;
15364        assert_eq!(entries.len(), 1);
15365        assert!(!entries[0]
15366            .disposition_detail
15367            .as_deref()
15368            .unwrap()
15369            .contains("registration_timeout"));
15370        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15371        actor.module.stop().await.unwrap();
15372    }
15373
15374    #[tokio::test(start_paused = true)]
15375    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15376        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15377        let pid = actor.module.status().unwrap().pid;
15378        tokio::time::advance(Duration::from_secs(119)).await;
15379        actor.roster.close();
15380        let turns = actor.turns();
15381        tokio::time::advance(Duration::from_secs(300)).await;
15382        let parked = actor.wait_for_select(|at| at.turn > turns).await;
15383        assert_eq!(parked.wake_after, None);
15384        assert_eq!(actor.module.status().unwrap().pid, pid);
15385        assert!(actor.module.terminal_history().entries.is_empty());
15386        actor.module.stop().await.unwrap();
15387        let status = actor.module.status().unwrap();
15388        assert_eq!(status.restart_count, 0);
15389        assert_eq!(
15390            actor.module.terminal_history().entries[0].disposition,
15391            TerminalDisposition::DaemonShutdown
15392        );
15393    }
15394
15395    #[tokio::test(start_paused = true)]
15396    async fn no_health_children_park_for_sixty_seconds() {
15397        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15398        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15399        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15400        unadvertised.hello(1, false);
15401        // Include any one-off registration wake in the budget; receiving a
15402        // no-health HELLO must not turn a parked child into a polling child.
15403        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
15404        // a single 60 s jump would only observe one expired timer.
15405        for _ in 0..6000 {
15406            tokio::time::advance(Duration::from_millis(10)).await;
15407            tokio::task::yield_now().await;
15408        }
15409        for actor in [&none, &unregistered, &unadvertised] {
15410            assert!(
15411                actor.turns() <= 5,
15412                "no-health actor woke {} times",
15413                actor.turns()
15414            );
15415            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15416        }
15417    }
15418
15419    #[tokio::test(start_paused = true)]
15420    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15421        // With a zero cadence the first probe is due at once, so this measures
15422        // only how fast the registration notification starts health probing,
15423        // not the normal 30-33 s wait before a first probe (checked below).
15424        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15425        let tick = Instant::now();
15426        actor.hello(1, true);
15427        actor.probe().await;
15428        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15429    }
15430
15431    #[tokio::test(start_paused = true)]
15432    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15433        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15434        let hello_tick = Instant::now();
15435        actor.hello(1, true);
15436        let armed = actor
15437            .wait_for_select(|checkpoint| {
15438                checkpoint.registered_connection == Some(ConnectionId::new(1))
15439                    && checkpoint.next_probe_at.is_some()
15440            })
15441            .await;
15442        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15443        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15444        let first_deadline = hello_tick + delay;
15445        assert_eq!(armed.next_probe_at, Some(first_deadline));
15446        tokio::time::advance(Duration::from_secs(29)).await;
15447        let turns = actor.turns();
15448        actor
15449            .registry
15450            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15451            .unwrap();
15452        let updated = actor
15453            .wait_for_select(|checkpoint| checkpoint.turn > turns)
15454            .await;
15455        assert_eq!(
15456            updated.next_probe_at,
15457            Some(first_deadline),
15458            "catalog update must not reset cadence"
15459        );
15460        actor.no_probe();
15461        tokio::time::advance(first_deadline - Instant::now()).await;
15462        let frame = actor.probe().await;
15463        actor.answer(1, frame);
15464        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15465        let next_deadline = Instant::now() + next;
15466        let rearmed = actor
15467            .wait_for_select(|checkpoint| {
15468                checkpoint
15469                    .next_probe_at
15470                    .is_some_and(|deadline| deadline > first_deadline)
15471            })
15472            .await;
15473        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15474        tokio::time::advance(next - Duration::from_nanos(1)).await;
15475        actor.no_probe();
15476        tokio::time::advance(Duration::from_nanos(1)).await;
15477        actor.probe().await;
15478    }
15479
15480    #[tokio::test(start_paused = true)]
15481    async fn disconnect_disarms_and_reconnect_rearms_health() {
15482        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15483        actor.hello(1, true);
15484        actor
15485            .wait_for_select(|checkpoint| {
15486                checkpoint.registered_connection == Some(ConnectionId::new(1))
15487                    && checkpoint.next_probe_at.is_some()
15488            })
15489            .await;
15490        let turns = actor.turns();
15491        let tick = Instant::now();
15492        actor
15493            .registry
15494            .deregister_connection(ConnectionId::new(1))
15495            .unwrap();
15496        let disarmed = actor
15497            .wait_for_select(|checkpoint| {
15498                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15499            })
15500            .await;
15501        assert_eq!(disarmed.next_probe_at, None);
15502        assert_eq!(disarmed.wake_after, None);
15503        assert_eq!(
15504            Instant::now(),
15505            tick,
15506            "disconnect must disarm in the notification tick"
15507        );
15508        tokio::time::advance(Duration::from_secs(60)).await;
15509        actor.no_probe();
15510        let reconnect_tick = Instant::now();
15511        actor.hello(2, true);
15512        let rearmed = actor
15513            .wait_for_select(|checkpoint| {
15514                checkpoint.registered_connection == Some(ConnectionId::new(2))
15515                    && checkpoint.next_probe_at.is_some()
15516            })
15517            .await;
15518        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15519        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15520        tokio::time::advance(delay).await;
15521        actor.probe().await;
15522    }
15523
15524    #[tokio::test(start_paused = true)]
15525    async fn reregistration_without_health_disarms_the_old_schedule() {
15526        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15527        actor.hello(1, true);
15528        actor
15529            .wait_for_select(|checkpoint| {
15530                checkpoint.registered_connection == Some(ConnectionId::new(1))
15531                    && checkpoint.next_probe_at.is_some()
15532            })
15533            .await;
15534        actor
15535            .registry
15536            .deregister_connection(ConnectionId::new(1))
15537            .unwrap();
15538        actor.hello(2, false);
15539        let disarmed = actor
15540            .wait_for_select(|checkpoint| {
15541                checkpoint.registered_connection == Some(ConnectionId::new(2))
15542            })
15543            .await;
15544        assert_eq!(disarmed.next_probe_at, None);
15545        assert_eq!(disarmed.wake_after, None);
15546        tokio::time::advance(Duration::from_secs(60)).await;
15547        actor.no_probe();
15548    }
15549
15550    #[tokio::test(start_paused = true)]
15551    async fn restart_waits_for_its_own_hello_before_rearming_health() {
15552        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15553        actor.hello(1, true);
15554        let armed = actor
15555            .wait_for_select(|checkpoint| {
15556                checkpoint.registered_connection == Some(ConnectionId::new(1))
15557                    && checkpoint.next_probe_at.is_some()
15558            })
15559            .await;
15560        // Simulate the old process's connection closing before the restart
15561        // tears the process down. The replacement process has started but has
15562        // not sent its HELLO, so it is not registered.
15563        actor
15564            .registry
15565            .deregister_connection(ConnectionId::new(1))
15566            .unwrap();
15567        actor.module.restart(Some(0)).await.unwrap();
15568        let parked = actor
15569            .wait_for_select(|checkpoint| {
15570                checkpoint.generation > armed.generation
15571                    && checkpoint.registered_connection.is_none()
15572                    && checkpoint.next_probe_at.is_none()
15573            })
15574            .await;
15575        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15576        actor.rx.close();
15577        // Drain the stop notice the restart queued for the old process's connection,
15578        // so it isn't mistaken for traffic to the replacement.
15579        while actor.rx.try_recv().is_ok() {}
15580        let turns = actor.turns();
15581        tokio::time::advance(Duration::from_secs(60)).await;
15582        let after = actor.turns();
15583        eprintln!(
15584            "restart park turns: before={turns} after={after} delta={}",
15585            after - turns
15586        );
15587        assert_eq!(after, turns, "replacement must park before HELLO");
15588        let hello_tick = Instant::now();
15589        actor.hello(2, true);
15590        let rearmed = actor
15591            .wait_for_select(|checkpoint| {
15592                checkpoint.registered_connection == Some(ConnectionId::new(2))
15593                    && checkpoint.next_probe_at.is_some()
15594            })
15595            .await;
15596        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15597        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15598        tokio::time::advance(delay).await;
15599        actor.probe().await;
15600    }
15601
15602    #[tokio::test(start_paused = true)]
15603    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15604        use tokio::{
15605            io::{AsyncReadExt, AsyncWriteExt},
15606            net::TcpListener,
15607        };
15608        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15609        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15610        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15611        let (started_tx, mut started_rx) = mpsc::channel(8);
15612        let peer = tokio::spawn(async move {
15613            loop {
15614                let (mut socket, _) = listener.accept().await.unwrap();
15615                let mut request = [0; 1024];
15616                assert!(socket.read(&mut request).await.unwrap() > 0);
15617                started_tx.send(Instant::now()).await.unwrap();
15618                socket
15619                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15620                    .await
15621                    .unwrap();
15622            }
15623        });
15624        actor
15625            .module
15626            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15627            .await
15628            .unwrap();
15629        let first_tick = Instant::now();
15630        let first = actor
15631            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15632            .await;
15633        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15634        assert!(started_rx.try_recv().is_err());
15635
15636        // Restart just before the old deadline: reusing it would give the new
15637        // process only one second to start, rather than a full cadence.
15638        tokio::time::advance(
15639            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15640        )
15641        .await;
15642        actor.module.restart(Some(0)).await.unwrap();
15643        let spawn_tick = Instant::now();
15644        let replacement = actor
15645            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15646            .await;
15647        let deadline = replacement.next_probe_at.unwrap();
15648        assert!(
15649            deadline >= spawn_tick + actor.health.cadence,
15650            "a replacement must not inherit its predecessor's probe deadline"
15651        );
15652        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15653        assert!(
15654            started_rx.try_recv().is_err(),
15655            "no HTTP probe during startup grace"
15656        );
15657        tokio::time::advance(deadline - Instant::now()).await;
15658        let landed = wait_for("first replacement HTTP probe", || {
15659            started_rx.try_recv().ok()
15660        })
15661        .await;
15662        assert!(landed >= spawn_tick + actor.health.cadence);
15663        wait_for("healthy replacement", || {
15664            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15665                .then_some(())
15666        })
15667        .await;
15668        peer.abort();
15669    }
15670
15671    #[tokio::test(start_paused = true)]
15672    async fn rescan_adds_and_removes_http_health_without_polling() {
15673        use tokio::{
15674            io::{AsyncReadExt, AsyncWriteExt},
15675            net::TcpListener,
15676        };
15677        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15678        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15679        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15680        // One millisecond separates successive probes without a polling delay.
15681        actor.health.cadence = Duration::from_millis(1);
15682        let (started_tx, mut started_rx) = mpsc::channel(8);
15683        let peer = tokio::spawn(async move {
15684            loop {
15685                let (mut socket, _) = listener.accept().await.unwrap();
15686                let mut request = [0; 1024];
15687                assert!(socket.read(&mut request).await.unwrap() > 0);
15688                started_tx.send(()).await.unwrap();
15689                socket
15690                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15691                    .await
15692                    .unwrap();
15693            }
15694        });
15695        let tick = Instant::now();
15696        let turns = actor.turns();
15697        actor
15698            .module
15699            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15700            .await
15701            .unwrap();
15702        let armed = actor
15703            .wait_for_select(|checkpoint| {
15704                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15705            })
15706            .await;
15707        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15708        assert_eq!(Instant::now(), tick);
15709        tokio::time::advance(Duration::from_millis(1)).await;
15710        let mut started = false;
15711        wait_for("successful HTTP probe", || {
15712            started |= started_rx.try_recv().is_ok();
15713            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15714                .then_some(())
15715        })
15716        .await;
15717        assert!(started, "added HTTP check must start at its first deadline");
15718        assert_eq!(
15719            actor.module.status().unwrap().health.status,
15720            SupervisorHealthStatus::Ok
15721        );
15722        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15723        actor.health.http = None;
15724        let turns = actor.turns();
15725        actor
15726            .module
15727            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15728            .await
15729            .unwrap();
15730        let parked = actor
15731            .wait_for_select(|checkpoint| {
15732                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15733            })
15734            .await;
15735        assert_eq!(parked.wake_after, None);
15736        let turns = actor.turns();
15737        tokio::time::advance(Duration::from_secs(60)).await;
15738        assert_eq!(
15739            actor.turns(),
15740            turns,
15741            "removed HTTP check must park the actor"
15742        );
15743        assert!(
15744            started_rx.try_recv().is_err(),
15745            "removed HTTP check must stay disarmed"
15746        );
15747        assert_eq!(
15748            actor.module.status().unwrap().health,
15749            ModuleHealthStatus::default()
15750        );
15751        peer.abort();
15752    }
15753}
15754
15755#[cfg(test)]
15756mod drain_busy_reprobe_tests {
15757    //! The drain's busy-gauge wait, driven against an in-process module
15758    //! connection whose health answers the test scripts. Time is paused, so
15759    //! every elapsed duration below is exact virtual time.
15760    use super::*;
15761    use std::collections::BTreeMap;
15762    use std::sync::Mutex as StdMutex;
15763    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15764
15765    const MODULE_ID: &str = "drain-reprobe-module";
15766    const CONNECTION: u64 = 7;
15767    const GAUGE: &str = "runs_in_flight";
15768    /// The production default for both the health cadence and the drain
15769    /// ceiling, which is the pairing that used to cost a full ceiling.
15770    const CADENCE: Duration = Duration::from_secs(30);
15771    const CEILING: Duration = Duration::from_secs(30);
15772
15773    /// How the scripted module answers each `health.check`.
15774    #[derive(Clone, Copy)]
15775    enum Answer {
15776        /// Report the gauge busy until this instant, quiet from then on, after
15777        /// taking `latency` to answer.
15778        BusyUntil { until: Instant, latency: Duration },
15779        /// Answer at once, with metrics that leave the declared gauge out.
15780        OmitsGauge,
15781        /// Never answer.
15782        Silent,
15783    }
15784
15785    struct Drain {
15786        forwarding: Arc<ForwardingTable>,
15787        runtime: SupervisorRuntimeConfig,
15788        endpoint: crate::ModuleEndpointId,
15789        /// When each probe reached the module.
15790        probes: Arc<StdMutex<Vec<Instant>>>,
15791        responder: tokio::task::JoinHandle<()>,
15792    }
15793
15794    impl Drain {
15795        fn start(answer: Answer) -> Self {
15796            let registry = Arc::new(Registry::default());
15797            let forwarding = Arc::new(ForwardingTable::default());
15798            let health = HealthConfig {
15799                cadence: CADENCE,
15800                ..HealthConfig::default()
15801            };
15802            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15803                .with_forwarding(forwarding.clone())
15804                .with_health_config(health)
15805                .runtime_config();
15806            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15807            let endpoint = forwarding
15808                .register_module_connection(
15809                    ConnectionId::new(CONNECTION),
15810                    MODULE_ID.to_string(),
15811                    subc_protocol::PROTOCOL_VERSION,
15812                    Concurrency::ModuleManaged,
15813                    FrameSink::new(tx),
15814                )
15815                .unwrap();
15816            let probes = Arc::new(StdMutex::new(Vec::new()));
15817            let responder = tokio::spawn({
15818                let forwarding = forwarding.clone();
15819                let probes = probes.clone();
15820                async move {
15821                    while let Some(frame) = rx.recv().await {
15822                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15823                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15824                        probes.lock().unwrap().push(Instant::now());
15825                        let metrics = match answer {
15826                            Answer::BusyUntil { until, latency } => {
15827                                sleep(latency).await;
15828                                let busy = u64::from(Instant::now() < until);
15829                                serde_json::json!({ GAUGE: busy })
15830                            }
15831                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15832                            Answer::Silent => continue,
15833                        };
15834                        // A probe abandoned at its deadline has no RPC left to
15835                        // complete, so a late answer may be refused.
15836                        let _ = forwarding.complete_module_control_rpc(
15837                            ConnectionId::new(CONNECTION),
15838                            frame.header.corr,
15839                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15840                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15841                                status: HealthStatus::Ok,
15842                                detail: None,
15843                                metrics: Some(metrics),
15844                            }),
15845                        );
15846                    }
15847                }
15848            });
15849            Self {
15850                forwarding,
15851                runtime,
15852                endpoint,
15853                probes,
15854                responder,
15855            }
15856        }
15857
15858        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15859        /// how long it took.
15860        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15861            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15862            (drained, elapsed)
15863        }
15864
15865        /// [`Self::run`], also returning what the drain's probes read.
15866        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15867            let started = Instant::now();
15868            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15869            let drained = wait_for_forwarding_quiescence(
15870                &self.forwarding,
15871                MODULE_ID,
15872                &self.runtime,
15873                self.endpoint,
15874                started + ceiling,
15875                &[GAUGE.to_string()],
15876                DrainScope::Active,
15877                &mut trace,
15878            )
15879            .await
15880            .unwrap();
15881            (drained, started.elapsed(), trace)
15882        }
15883
15884        fn probes(&self) -> Vec<Instant> {
15885            self.probes.lock().unwrap().clone()
15886        }
15887    }
15888
15889    impl Drop for Drain {
15890        fn drop(&mut self) {
15891            self.responder.abort();
15892        }
15893    }
15894
15895    #[tokio::test(start_paused = true)]
15896    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15897        let drain = Drain::start(Answer::BusyUntil {
15898            until: Instant::now() + Duration::from_secs(1),
15899            latency: Duration::ZERO,
15900        });
15901        let (drained, elapsed) = drain.run(CEILING).await;
15902        assert!(
15903            drained,
15904            "gauges that read zero during the drain must let it finish drained, \
15905             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15906        );
15907        assert!(
15908            elapsed < Duration::from_secs(3),
15909            "drain took {elapsed:?} after the gauges went quiet at 1s"
15910        );
15911        let probes = drain.probes();
15912        assert!(
15913            probes.len() >= 2,
15914            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15915        );
15916    }
15917
15918    /// The same shape in real time on a multi-threaded runtime: the first
15919    /// probe is answered busy at once, within a millisecond of the drain
15920    /// starting, and later probes read idle. The drain must keep probing
15921    /// after that first instant answer and end soon after the gauge falls.
15922    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15923    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15924        let drain = Drain::start(Answer::BusyUntil {
15925            until: Instant::now() + Duration::from_millis(600),
15926            latency: Duration::ZERO,
15927        });
15928        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15929        assert!(drained, "the gauge fell at 600ms: {trace}");
15930        assert!(
15931            elapsed < Duration::from_secs(3),
15932            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15933        );
15934        let probes = drain.probes();
15935        assert!(
15936            probes.len() >= 2 && trace.probes as usize == probes.len(),
15937            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15938        );
15939    }
15940
15941    #[tokio::test(start_paused = true)]
15942    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15943        let started = Instant::now();
15944        let drain = Drain::start(Answer::BusyUntil {
15945            until: started + Duration::from_secs(3600),
15946            latency: Duration::ZERO,
15947        });
15948        let (drained, elapsed) = drain.run(CEILING).await;
15949        assert!(!drained, "a module busy throughout must not drain");
15950        assert!(
15951            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15952            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15953        );
15954        // Re-probing is paced by the interval, not a busy loop: at most one
15955        // probe per interval across the ceiling, plus the first at time zero.
15956        let probes = drain.probes();
15957        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15958        assert!(
15959            probes.len() as u128 <= max_probes,
15960            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15961            probes.len()
15962        );
15963        for pair in probes.windows(2) {
15964            assert!(
15965                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15966                "consecutive probes {:?} apart",
15967                pair[1] - pair[0]
15968            );
15969        }
15970    }
15971
15972    #[tokio::test(start_paused = true)]
15973    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15974        // Each answer takes longer than the re-probe interval; the next probe
15975        // must wait for the previous answer and then the interval.
15976        let latency = Duration::from_secs(1);
15977        let drain = Drain::start(Answer::BusyUntil {
15978            until: Instant::now() + Duration::from_secs(3600),
15979            latency,
15980        });
15981        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15982        assert!(!drained);
15983        let probes = drain.probes();
15984        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15985        for pair in probes.windows(2) {
15986            assert!(
15987                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15988                "a probe started {:?} after the previous one, before it could finish",
15989                pair[1] - pair[0]
15990            );
15991        }
15992    }
15993
15994    #[tokio::test(start_paused = true)]
15995    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15996        let drain = Drain::start(Answer::Silent);
15997        let ceiling = Duration::from_secs(2);
15998        let (drained, elapsed) = drain.run(ceiling).await;
15999        assert!(!drained, "an unanswered probe counts as busy");
16000        assert!(
16001            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
16002            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
16003        );
16004        assert_eq!(
16005            drain.probes().len(),
16006            1,
16007            "the unanswered probe holds the only slot until the ceiling"
16008        );
16009    }
16010
16011    /// Every event logged while it is installed, as `message` plus fields.
16012    #[derive(Clone, Default)]
16013    struct LogCapture {
16014        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16015    }
16016
16017    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16018        fn on_event(
16019            &self,
16020            event: &tracing::Event<'_>,
16021            _context: tracing_subscriber::layer::Context<'_, S>,
16022        ) {
16023            struct Fields(BTreeMap<String, String>);
16024            impl tracing::field::Visit for Fields {
16025                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16026                    self.0
16027                        .insert(field.name().to_string(), format!("{value:?}"));
16028                }
16029            }
16030            let mut fields = Fields(BTreeMap::new());
16031            event.record(&mut fields);
16032            self.events.lock().unwrap().push(fields.0);
16033        }
16034    }
16035
16036    impl LogCapture {
16037        fn install(&self) -> tracing::subscriber::DefaultGuard {
16038            use tracing_subscriber::layer::SubscriberExt;
16039            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16040        }
16041
16042        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16043            self.events
16044                .lock()
16045                .unwrap()
16046                .iter()
16047                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16048                .cloned()
16049                .collect()
16050        }
16051    }
16052
16053    /// A module that declares a busy gauge but leaves it out of its health
16054    /// answers holds a drain to its ceiling. The timeout line must say so by
16055    /// name, and the drain must warn about the missing gauge once, not on
16056    /// every probe.
16057    #[tokio::test(start_paused = true)]
16058    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16059        let capture = LogCapture::default();
16060        let _guard = capture.install();
16061        let drain = Drain::start(Answer::OmitsGauge);
16062        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16063        assert!(!drained, "a gauge never reported cannot read quiet");
16064        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16065
16066        let missing = capture.with_message("lacks a declared busy gauge");
16067        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16068        assert_eq!(
16069            missing[0].get("missing_gauges").map(String::as_str),
16070            Some(GAUGE)
16071        );
16072
16073        warn_drain_timed_out(
16074            MODULE_ID,
16075            Duration::from_secs(2),
16076            RouteCloseReason::Restart,
16077            &Default::default(),
16078            &trace,
16079        );
16080        let timeouts = capture.with_message("route drain timed out");
16081        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16082        let gauges = &timeouts[0]["busy_gauges"];
16083        assert!(
16084            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16085            "the timeout line must name the gauge and why it read busy: {gauges}"
16086        );
16087        assert!(
16088            gauges.ends_with("; no value ever read"),
16089            "no value was ever read for the gauge: {gauges}"
16090        );
16091        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16092    }
16093
16094    /// The trace names a busy value and when it was read. For a probe that
16095    /// never got an answer, it names the probe's error kind and says that no
16096    /// value was ever read.
16097    #[tokio::test(start_paused = true)]
16098    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16099        let busy = Drain::start(Answer::BusyUntil {
16100            until: Instant::now() + Duration::from_secs(3600),
16101            latency: Duration::ZERO,
16102        });
16103        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16104        let rendered = trace.to_string();
16105        assert!(
16106            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16107                && rendered.contains("; last value non_zero(1) @"),
16108            "{rendered}"
16109        );
16110
16111        let silent = Drain::start(Answer::Silent);
16112        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16113        // The probe is capped at the 1 s ceiling, and its reading is stamped
16114        // when it gave up.
16115        assert_eq!(
16116            trace.to_string(),
16117            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16118        );
16119        assert_eq!(trace.probes, 1);
16120    }
16121
16122    /// A gauge value that is not a count reads busy and is shown as sent.
16123    #[test]
16124    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16125        let gauges = [GAUGE.to_string()];
16126        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16127        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16128        assert_eq!(
16129            busy_gauge_observation(&verdicts),
16130            BusyGaugeObservation::Busy
16131        );
16132        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16133        assert_eq!(
16134            busy_gauge_observation(&zero),
16135            BusyGaugeObservation::Quiescent
16136        );
16137        let none = gauge_verdicts(None, &gauges);
16138        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16139    }
16140}
16141
16142#[cfg(test)]
16143mod health_evidence_tests {
16144    use super::{HealthProbeError, HealthProbeEvidence};
16145    use std::collections::HashSet;
16146
16147    /// The evidential asymmetry, asserted rather than described.
16148    ///
16149    /// Exactly ONE observation is proof a module cannot serve, and the one that
16150    /// fires under CPU starvation is not it. Before the split, all fifteen
16151    /// construction sites collapsed into a single String, so a timeout carried the
16152    /// same weight as a dead lane -- which is how a healthy module was restarted
16153    /// three times in one day.
16154    #[test]
16155    fn only_a_dead_lane_is_proof_of_death() {
16156        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16157        // Three non-proof classes, each for a different reason: silence is
16158        // consistent with health, a bad answer proves the module ALIVE, and a
16159        // daemon-side fault never reached the module at all.
16160        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16161        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16162        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16163    }
16164
16165    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
16166    ///
16167    /// A shared label renders two different observations identically in the line an
16168    /// operator reads after an unexplained restart -- the exact confusion this
16169    /// change removes.
16170    #[test]
16171    fn every_evidence_class_has_a_distinct_label() {
16172        let labels = [
16173            HealthProbeError::lane_dead("").label(),
16174            HealthProbeError::no_answer("").label(),
16175            HealthProbeError::bad_answer("").label(),
16176            HealthProbeError::misconfigured("").label(),
16177        ];
16178        let unique: HashSet<_> = labels.iter().collect();
16179        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16180    }
16181
16182    /// The class is additional information, not a replacement.
16183    ///
16184    /// An operator needs both "this was silence" and the specific text saying how
16185    /// long we waited; a classification that swallowed the message would trade one
16186    /// missing distinction for another.
16187    #[test]
16188    fn classification_preserves_the_original_message() {
16189        let err = HealthProbeError::no_answer("module did not answer within 5s");
16190        assert_eq!(err.to_string(), "module did not answer within 5s");
16191        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16192    }
16193}
16194
16195#[cfg(test)]
16196mod health_tombstone_tests {
16197    use std::{path::PathBuf, sync::Arc, time::Duration};
16198
16199    use subc_protocol::{
16200        manifest::Concurrency,
16201        session::{HealthStatus, ModuleControlResponse},
16202    };
16203    use tokio::sync::mpsc;
16204
16205    use super::{
16206        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16207        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16208    };
16209    use crate::{
16210        control::ControlHandler,
16211        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16212        registry::{ConnectionId, Registry},
16213        router::FrameSink,
16214    };
16215
16216    struct ProbeHarness {
16217        spec: ModuleSpec,
16218        runtime: SupervisorRuntimeConfig,
16219        forwarding: Arc<ForwardingTable>,
16220        module_connection: ConnectionId,
16221        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16222        handler: ControlHandler,
16223        module: super::SupervisedModule,
16224    }
16225
16226    fn probe_harness() -> ProbeHarness {
16227        let registry = Arc::new(Registry::default());
16228        let forwarding = Arc::new(ForwardingTable::default());
16229        let supervisor_handle = super::SupervisorHandle::new();
16230        let health = HealthConfig {
16231            http: None,
16232            cadence: Duration::from_secs(30),
16233            deadline: Duration::from_secs(5),
16234            failure_threshold: 3,
16235            on_degraded: HealthAction::Report,
16236            on_failing: HealthAction::Report,
16237            critical: false,
16238        };
16239        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
16240            .with_forwarding(Arc::clone(&forwarding))
16241            .with_handle(supervisor_handle.clone())
16242            .with_health_config(health);
16243        let spec = ModuleSpec {
16244            module_id: "late-health-module".to_string(),
16245            program: PathBuf::from("disabled-module"),
16246            args: Vec::new(),
16247            env: Vec::new(),
16248            reserved: false,
16249            reserved_prefixes: Vec::new(),
16250            protocol: ModuleProtocol::Subc,
16251            overlap: Default::default(),
16252            carry_containment: false,
16253        };
16254        let module = supervisor
16255            .supervise_configured(spec.clone(), false)
16256            .unwrap();
16257        let runtime = supervisor.runtime_config();
16258        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16259            .with_supervisor(supervisor_handle);
16260        let module_connection = ConnectionId::new(700);
16261        let (module_tx, module_rx) = mpsc::channel(8);
16262        forwarding
16263            .register_module_connection(
16264                module_connection,
16265                spec.module_id.clone(),
16266                subc_protocol::PROTOCOL_VERSION,
16267                Concurrency::ModuleManaged,
16268                FrameSink::new(module_tx),
16269            )
16270            .unwrap();
16271
16272        ProbeHarness {
16273            spec,
16274            runtime,
16275            forwarding,
16276            module_connection,
16277            module_rx,
16278            handler,
16279            module,
16280        }
16281    }
16282
16283    async fn finish_after(
16284        harness: &mut ProbeHarness,
16285        stall: Duration,
16286    ) -> ModuleControlRpcCompletion {
16287        assert!(stall > harness.runtime.health.deadline);
16288        let deadline = harness.runtime.health.deadline;
16289        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16290        let answer = async {
16291            let frame = harness.module_rx.recv().await.expect("health.check frame");
16292            tokio::time::advance(deadline).await;
16293            tokio::task::yield_now().await;
16294            tokio::time::advance(stall - deadline).await;
16295            harness
16296                .forwarding
16297                .complete_module_control_rpc(
16298                    harness.module_connection,
16299                    frame.header.corr,
16300                    Some("health.check"),
16301                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16302                        status: HealthStatus::Ok,
16303                        detail: None,
16304                        metrics: None,
16305                    }),
16306                )
16307                .unwrap()
16308        };
16309        let (probe_result, completion) = tokio::join!(probe, answer);
16310        let err = probe_result.expect_err("probe must miss its deadline");
16311        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16312        completion
16313    }
16314
16315    async fn time_out_without_answer(harness: &mut ProbeHarness) {
16316        let deadline = harness.runtime.health.deadline;
16317        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16318        let exhaust_deadline = async {
16319            let _frame = harness.module_rx.recv().await.expect("health.check frame");
16320            tokio::time::advance(deadline).await;
16321            tokio::task::yield_now().await;
16322        };
16323        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16324        let err = probe_result.expect_err("probe must miss its deadline");
16325        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16326    }
16327
16328    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16329        let registry = Arc::clone(&harness.module.inner.registry);
16330        let snapshot = Arc::clone(&harness.module.inner.snapshot);
16331        let process_liveness = super::SupervisorProcessLiveness::default();
16332        let mut child = None;
16333        let cycle = super::run_health_probe_cycle(
16334            &harness.spec,
16335            &harness.runtime,
16336            &registry,
16337            &process_liveness,
16338            &snapshot,
16339            &mut child,
16340        );
16341        let peer = async {
16342            let frame = harness.module_rx.recv().await.expect("health.check frame");
16343            if answer {
16344                harness
16345                    .forwarding
16346                    .complete_module_control_rpc(
16347                        harness.module_connection,
16348                        frame.header.corr,
16349                        Some("health.check"),
16350                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16351                            status: HealthStatus::Ok,
16352                            detail: None,
16353                            metrics: Some(serde_json::json!({"ready": true})),
16354                        }),
16355                    )
16356                    .unwrap();
16357            } else {
16358                tokio::time::advance(harness.runtime.health.deadline).await;
16359                tokio::task::yield_now().await;
16360            }
16361        };
16362        tokio::join!(cycle, peer);
16363    }
16364
16365    #[tokio::test(start_paused = true)]
16366    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16367        let mut harness = probe_harness();
16368        // Drive the probe cycle directly with an in-memory wire peer. Stop the
16369        // disabled module's monitor so only this test owns lifecycle transitions;
16370        // no OS process is launched, and a restart is observed at scheduling.
16371        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16372        monitor.abort();
16373        let _ = monitor.await;
16374        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16375            state.enabled = true;
16376            state.state = super::ModuleState::Running;
16377            state.process_alive = true;
16378        })
16379        .unwrap();
16380
16381        run_probe_cycle(&mut harness, true).await;
16382        assert_eq!(
16383            harness.module.status().unwrap().health.status,
16384            super::SupervisorHealthStatus::Ok
16385        );
16386
16387        for failures in 1..harness.runtime.health.failure_threshold {
16388            run_probe_cycle(&mut harness, false).await;
16389            let status = harness.module.status().unwrap();
16390            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16391            assert_eq!(status.health.consecutive_failures, failures);
16392            assert!(status.health.last_probe_ms.is_some());
16393            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16394            assert!(status.health.metrics.is_none());
16395            assert_eq!(status.state, super::ModuleState::Running);
16396            assert!(status.process_alive);
16397            assert_eq!(status.restart_count, 0);
16398            assert_eq!(status.lifetime_restarts, 0);
16399            assert!(status.health.last_action.is_none());
16400            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16401        }
16402
16403        run_probe_cycle(&mut harness, true).await;
16404        let recovered = harness.module.status().unwrap();
16405        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16406        assert_eq!(recovered.health.consecutive_failures, 0);
16407        assert!(recovered.health.detail.is_none());
16408        assert_eq!(
16409            recovered.health.metrics,
16410            Some(serde_json::json!({"ready": true}))
16411        );
16412        assert_eq!(recovered.lifetime_restarts, 0);
16413
16414        for failures in 1..=harness.runtime.health.failure_threshold {
16415            run_probe_cycle(&mut harness, false).await;
16416            let status = harness.module.status().unwrap();
16417            assert_eq!(status.health.consecutive_failures, failures);
16418            if failures < harness.runtime.health.failure_threshold {
16419                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16420                assert_eq!(status.state, super::ModuleState::Running);
16421                assert_eq!(status.lifetime_restarts, 0);
16422                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16423            } else {
16424                assert_eq!(
16425                    status.health.status,
16426                    super::SupervisorHealthStatus::Unresponsive
16427                );
16428                assert_eq!(status.state, super::ModuleState::Restarting);
16429                assert_eq!(status.restart_count, 1);
16430                assert_eq!(status.lifetime_restarts, 1);
16431                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16432                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16433            }
16434        }
16435    }
16436
16437    #[tokio::test(start_paused = true)]
16438    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16439        let mut harness = probe_harness();
16440
16441        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16442        let first_latency = match &first {
16443            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16444            other => panic!("late answer was not retained: {other:?}"),
16445        };
16446        assert!(harness.handler.observe_module_control_completion(first));
16447
16448        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16449        let second_latency = match &second {
16450            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16451            other => panic!("late answer was not retained: {other:?}"),
16452        };
16453        assert!(harness.handler.observe_module_control_completion(second));
16454
16455        assert_eq!(first_latency, Duration::from_secs(8));
16456        assert_eq!(
16457            second_latency - first_latency,
16458            Duration::from_secs(3),
16459            "latency must grow linearly with the additional stall"
16460        );
16461        let health = harness.module.status().unwrap().health;
16462        assert_eq!(health.late_answer_count, 2);
16463        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16464    }
16465
16466    /// A module that answers every probe late must never march to the kill
16467    /// threshold: the late answer proves it is alive, so it must clear the miss
16468    /// streak the timeout recorded. Without the reset, a CPU-starved module
16469    /// that serves every probe seconds past the deadline accumulates
16470    /// `consecutive_failures` to the threshold and is killed — the exact
16471    /// sequence from the 2026-08-14 aft disable, where the daemon logged
16472    /// "proves the module is alive" five times while counting five misses.
16473    #[tokio::test(start_paused = true)]
16474    async fn late_answer_clears_the_consecutive_failure_streak() {
16475        let mut harness = probe_harness();
16476
16477        // Timeout recorded first: the probe path saw no answer in time.
16478        time_out_without_answer(&mut harness).await;
16479        harness
16480            .module
16481            .record_health_probe_failure_for_test("[no-answer] test miss")
16482            .unwrap();
16483        assert_eq!(
16484            harness.module.status().unwrap().health.consecutive_failures,
16485            1,
16486            "precondition: the miss must be on the streak before the late answer"
16487        );
16488
16489        // The stalled reply then lands: proof of life.
16490        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16491        assert!(matches!(
16492            late,
16493            ModuleControlRpcCompletion::LateHealthAnswer { .. }
16494        ));
16495        assert!(harness.handler.observe_module_control_completion(late));
16496
16497        let health = harness.module.status().unwrap().health;
16498        assert_eq!(
16499            health.consecutive_failures, 0,
16500            "a late answer is an answer: the streak must reset"
16501        );
16502        assert_eq!(health.late_answer_count, 1);
16503    }
16504
16505    #[tokio::test(start_paused = true)]
16506    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16507        let mut harness = probe_harness();
16508
16509        for _ in 0..20 {
16510            time_out_without_answer(&mut harness).await;
16511            assert_eq!(
16512                harness.forwarding.health_probe_tombstone_count().unwrap(),
16513                1
16514            );
16515        }
16516    }
16517}
16518
16519#[cfg(test)]
16520mod child_env_tests {
16521    #[cfg(windows)]
16522    use super::{
16523        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16524    };
16525    #[cfg(windows)]
16526    use std::sync::{Arc, OnceLock};
16527    #[cfg(windows)]
16528    #[test]
16529    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16530        let spec = spec(Vec::new());
16531        let mut command = Command::new("unused");
16532        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16533            .unwrap()
16534            .unwrap();
16535        let name = command
16536            .as_std()
16537            .get_envs()
16538            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16539            .and_then(|(_, value)| value)
16540            .unwrap();
16541        assert_eq!(name, OsStr::new(handoff.name()));
16542        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16543        assert!(command.as_std().get_envs().any(|(key, value)| key
16544            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16545            && value == Some(OsStr::new("env"))));
16546        assert!(command
16547            .as_std()
16548            .get_envs()
16549            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16550    }
16551
16552    #[cfg(windows)]
16553    #[test]
16554    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16555        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16556        snapshot.process_alive = true;
16557        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16558        assert_eq!(
16559            observed_launch_nonce_source(&snapshot).as_deref(),
16560            Some("env")
16561        );
16562        let consumed = Arc::new(OnceLock::new());
16563        snapshot.pipe_consumed = Some(consumed.clone());
16564        assert_eq!(
16565            observed_launch_nonce_source(&snapshot).as_deref(),
16566            Some("env")
16567        );
16568        consumed.set(()).unwrap();
16569        assert_eq!(
16570            observed_launch_nonce_source(&snapshot).as_deref(),
16571            Some("pipe")
16572        );
16573        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16574        assert_eq!(
16575            observed_launch_nonce_source(&snapshot).as_deref(),
16576            Some("env")
16577        );
16578        clear_current_process_facts(&mut snapshot);
16579        assert_eq!(observed_launch_nonce_source(&snapshot), None);
16580    }
16581    use super::{
16582        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16583        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16584        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16585    };
16586    use std::{ffi::OsStr, path::PathBuf};
16587    use tokio::process::Command;
16588
16589    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16590        ModuleSpec {
16591            module_id: "env-plan".to_string(),
16592            program: PathBuf::from("/nonexistent"),
16593            args: Vec::new(),
16594            env,
16595            reserved: false,
16596            reserved_prefixes: Vec::new(),
16597            protocol: ModuleProtocol::Subc,
16598            overlap: Default::default(),
16599            carry_containment: false,
16600        }
16601    }
16602
16603    #[test]
16604    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16605        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16606        let digest = LaunchSpecDigest::from_spec(&base);
16607        let mut reordered = base.clone();
16608        reordered.env.reverse();
16609        reordered.module_id = "different-id".into();
16610        reordered.reserved = true;
16611        reordered.overlap = Default::default();
16612        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16613        let mut delimiter_collision = base.clone();
16614        delimiter_collision.env[0] = ("A=B".into(), "C".into());
16615        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16616        let mut changed = base.clone();
16617        changed.protocol = ModuleProtocol::None;
16618        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16619        changed = base.clone();
16620        changed.args.push("--flag".into());
16621        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16622        changed = base.clone();
16623        changed.program = PathBuf::from("/other");
16624        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16625        changed = base.clone();
16626        changed.env[1].1 = "other".into();
16627        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16628        assert!(!format!("{digest:?}").contains("B=C"));
16629        assert!(!format!("{digest:?}").contains("[u8"));
16630    }
16631
16632    #[cfg(unix)]
16633    #[test]
16634    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16635        use std::os::unix::ffi::OsStringExt;
16636        let mut left = spec(Vec::new());
16637        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16638        let mut right = left.clone();
16639        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16640        assert_eq!(
16641            left.program.to_string_lossy(),
16642            right.program.to_string_lossy()
16643        );
16644        assert_ne!(
16645            LaunchSpecDigest::from_spec(&left),
16646            LaunchSpecDigest::from_spec(&right)
16647        );
16648    }
16649
16650    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
16651    /// one still gets its own.
16652    ///
16653    /// This is the narrow goal `env_clear()` was reached for, and the reason the
16654    /// fix is `env_remove` rather than deleting the line: an operator's ambient
16655    /// filter silently becoming an unconfigured module's log level is a real
16656    /// defect, just a much smaller one than clearing the environment.
16657    ///
16658    /// Asserted on the command plan rather than a spawned child because proving
16659    /// the ABSENCE of an inherited variable needs the parent's environment
16660    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
16661    /// removal as `(key, None)`, which is exactly the distinction wanted: not
16662    /// "absent because nobody set it" but "explicitly unset for the child".
16663    #[test]
16664    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16665        let mut command = Command::new("/nonexistent");
16666        apply_child_env(&mut command, &spec(Vec::new()));
16667        let removed = command
16668            .as_std()
16669            .get_envs()
16670            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16671        assert!(
16672            removed,
16673            "ambient CK_LOG must be explicitly removed for an unconfigured module"
16674        );
16675
16676        let mut configured = Command::new("/nonexistent");
16677        apply_child_env(
16678            &mut configured,
16679            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16680        );
16681        let effective = configured
16682            .as_std()
16683            .get_envs()
16684            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16685            .last()
16686            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16687        assert_eq!(
16688            effective,
16689            Some(Some("debug".to_string())),
16690            "a module's configured CK_LOG must survive the ambient removal"
16691        );
16692    }
16693
16694    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
16695    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
16696    /// the same reason as the CK_LOG test above.
16697    ///
16698    /// The argument is the load-bearing half: a stock binary exits on an
16699    /// unknown flag before it listens, so with `--subc` appended the mode
16700    /// could not supervise the one process it exists for. Found by the first
16701    /// conformance run (nats-server: `flag provided but not defined: -subc`).
16702    #[test]
16703    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16704        let connection_file = std::path::Path::new("/run/subc-connection.json");
16705        let handle = SupervisorHandle::new();
16706
16707        let mut none_spec = spec(Vec::new());
16708        none_spec.protocol = ModuleProtocol::None;
16709        let mut none = Command::new("/nonexistent");
16710        let none_handoff =
16711            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16712                .expect("protocol-none spawn args apply");
16713        assert!(
16714            none_handoff.is_none(),
16715            "protocol:none spawn must not receive a nonce descriptor"
16716        );
16717        #[cfg(windows)]
16718        assert!(!none.as_std().get_envs().any(|(key, value)| key
16719            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16720            && value.is_some()));
16721        assert!(
16722            !none.as_std().get_envs().any(|(key, value)| key
16723                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16724                && value.is_some()),
16725            "protocol:none spawn must not name a nonce descriptor"
16726        );
16727        let none_args: Vec<String> = none
16728            .as_std()
16729            .get_args()
16730            .map(|a| a.to_string_lossy().into_owned())
16731            .collect();
16732        assert!(
16733            !none_args.iter().any(|a| a == SUBC_ARG),
16734            "protocol:none argv must not carry --subc; got {none_args:?}"
16735        );
16736        let none_has_nonce = none
16737            .as_std()
16738            .get_envs()
16739            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16740        assert!(
16741            !none_has_nonce,
16742            "protocol:none spawn must not receive a launch nonce"
16743        );
16744        let none_has_module_id = none
16745            .as_std()
16746            .get_envs()
16747            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16748        assert!(
16749            none_has_module_id,
16750            "SUBC_MODULE_ID is inert and stays on every path"
16751        );
16752        assert!(
16753            handle.spawn_nonce(&none_spec.module_id).is_none(),
16754            "no nonce record for a process that will never present one"
16755        );
16756
16757        // Control: the subc-wire path is unchanged by the branch above.
16758        let wire_spec = spec(Vec::new());
16759        let mut wire = Command::new("/nonexistent");
16760        let wire_handoff =
16761            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16762                .expect("subc-wire spawn args apply");
16763        let wire_fd_env = wire
16764            .as_std()
16765            .get_envs()
16766            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16767            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16768        #[cfg(unix)]
16769        assert_eq!(
16770            wire_fd_env,
16771            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16772            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16773        );
16774        #[cfg(windows)]
16775        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16776        #[cfg(not(any(unix, windows)))]
16777        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16778        let wire_args: Vec<String> = wire
16779            .as_std()
16780            .get_args()
16781            .map(|a| a.to_string_lossy().into_owned())
16782            .collect();
16783        assert_eq!(
16784            wire_args,
16785            vec![
16786                SUBC_ARG.to_string(),
16787                connection_file.to_string_lossy().into_owned()
16788            ],
16789            "a subc-wire spawn still carries --subc <path>"
16790        );
16791        assert_eq!(
16792            wire.as_std()
16793                .get_envs()
16794                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16795            !cfg!(unix),
16796            "only Windows supplies the environment nonce"
16797        );
16798        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16799    }
16800
16801    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16802    /// spec tries to set it; only a swap candidate carries it.
16803    ///
16804    /// "Set it only on candidates" is not enough, because spawn applies the
16805    /// spec's env verbatim and the daemon's own environment is inherited: either
16806    /// could hand a plain restart the swap role, and a module reading it would
16807    /// warm on its long swap budget while callers wait. Asserted as an explicit
16808    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16809    /// test above gives.
16810    #[test]
16811    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16812        let role = |command: &Command| {
16813            command
16814                .as_std()
16815                .get_envs()
16816                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16817                .last()
16818                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16819        };
16820        let forged = spec(vec![(
16821            SUBC_SPAWN_ROLE_ENV.to_string(),
16822            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16823        )]);
16824
16825        let mut plain = Command::new("/nonexistent");
16826        apply_child_env(&mut plain, &forged);
16827        apply_spawn_role(&mut plain, SpawnRole::Plain);
16828        assert_eq!(
16829            role(&plain),
16830            Some(None),
16831            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16832        );
16833
16834        let mut candidate = Command::new("/nonexistent");
16835        apply_child_env(&mut candidate, &spec(Vec::new()));
16836        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16837        assert_eq!(
16838            role(&candidate),
16839            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16840        );
16841    }
16842
16843    /// Daemon-private capture retention keys never reach the child.
16844    ///
16845    /// cortexkit-log exposes retention as a Rust struct with no environment
16846    /// names, so these entries are supervisor metadata. Passing them through
16847    /// would invent a public child-process contract by accident.
16848    #[test]
16849    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16850        let mut command = Command::new("/nonexistent");
16851        apply_child_env(
16852            &mut command,
16853            &spec(vec![
16854                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16855                ("KEPT".to_string(), "yes".to_string()),
16856            ]),
16857        );
16858        let keys: Vec<String> = command
16859            .as_std()
16860            .get_envs()
16861            .filter(|(_, value)| value.is_some())
16862            .map(|(key, _)| key.to_string_lossy().into_owned())
16863            .collect();
16864        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16865        assert!(
16866            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16867            "daemon-private capture key leaked to the child: {keys:?}"
16868        );
16869    }
16870}
16871
16872#[cfg(test)]
16873mod jitter_tests {
16874    use super::jittered_health_delay;
16875    use std::{collections::HashSet, time::Duration};
16876
16877    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16878    /// that actually occur rather than invented ones.
16879    ///
16880    /// This is a SAMPLE, not a registry: the property under test is that distinct
16881    /// ids disperse, which holds for any set of distinct strings. Several entries
16882    /// are already historical (modules get renamed), and that costs nothing here --
16883    /// but it means a reader must not mistake this for the live module set, and a
16884    /// rename sweep will match it without there being anything to change.
16885    const FLEET: [&str; 14] = [
16886        "aft",
16887        "alfonso-core",
16888        "magic-context",
16889        "broca",
16890        "thalamus",
16891        "quota",
16892        "engram",
16893        "plexus",
16894        "cerebellum",
16895        "astrocyte",
16896        "synapse",
16897        "subc-mcp",
16898        "cortexkit-credentials",
16899        "subc-federation",
16900    ];
16901
16902    /// Probes must not converge after a fleet-wide restart.
16903    ///
16904    /// This is the property the jitter exists for: every module reconnects at
16905    /// once, and without dispersal all fourteen would then probe on the same
16906    /// tick forever. Nothing failed visibly when this went untested -- a
16907    /// convergent fleet still probes correctly, just in a burst, so the symptom
16908    /// is a periodic load spike that looks like whatever else is running.
16909    #[test]
16910    fn probe_delays_disperse_across_the_fleet() {
16911        let cadence = Duration::from_secs(30);
16912        let delays: HashSet<Duration> = FLEET
16913            .iter()
16914            .map(|id| jittered_health_delay(id, 0, cadence))
16915            .collect();
16916        assert_eq!(
16917            delays.len(),
16918            FLEET.len(),
16919            "every supervised module must land on its own probe offset"
16920        );
16921    }
16922
16923    /// The offset may only ever DELAY a probe, never bring it forward.
16924    ///
16925    /// A delay below the cadence would probe a module more often than
16926    /// configured, which is the opposite of what an operator asked for and
16927    /// would tighten the failure budget without anyone changing it.
16928    #[test]
16929    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16930        let cadence = Duration::from_secs(30);
16931        let span = cadence / 10;
16932        for id in FLEET {
16933            for probe_index in 0..8 {
16934                let delay = jittered_health_delay(id, probe_index, cadence);
16935                assert!(
16936                    delay >= cadence,
16937                    "{id}#{probe_index}: jitter must not shorten the cadence"
16938                );
16939                assert!(
16940                    delay < cadence + span,
16941                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16942                );
16943            }
16944        }
16945    }
16946
16947    /// A module keeps its offset across daemon restarts.
16948    ///
16949    /// The delay is derived rather than randomised precisely so a restart does
16950    /// not re-roll every module into a fresh chance of collision. A random
16951    /// source would satisfy the dispersal test above and quietly lose this.
16952    #[test]
16953    fn a_module_offset_is_stable_across_restarts() {
16954        let cadence = Duration::from_secs(30);
16955        for id in FLEET {
16956            assert_eq!(
16957                jittered_health_delay(id, 0, cadence),
16958                jittered_health_delay(id, 0, cadence),
16959                "{id}: the same module and probe index must produce the same offset"
16960            );
16961        }
16962    }
16963
16964    /// A zero cadence disables probing rather than producing a busy loop.
16965    #[test]
16966    fn zero_cadence_yields_zero_delay() {
16967        assert_eq!(
16968            jittered_health_delay("aft", 0, Duration::ZERO),
16969            Duration::ZERO
16970        );
16971    }
16972}
16973
16974#[cfg(all(test, target_os = "linux"))]
16975mod cgroup_placement_tests {
16976    use super::{
16977        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16978        SuperviseError, SupervisedChild,
16979    };
16980    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16981    use cortexkit_test_support::ScratchDir;
16982    use std::{
16983        fs, io,
16984        path::{Path, PathBuf},
16985        sync::{Arc, Mutex},
16986    };
16987    use tokio::process::Command;
16988
16989    #[tokio::test]
16990    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16991        use super::*;
16992        let dir = ScratchDir::new("unique-spawn-cgroups");
16993        let root = PathBuf::from(format!(
16994            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16995            std::process::id(),
16996            unix_ms_now()
16997        ));
16998        if let Err(error) = fs::create_dir(&root) {
16999            assert!(
17000                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17001                "required cgroup test cannot execute: {error}"
17002            );
17003            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
17004            return;
17005        }
17006        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
17007        let group_count = || {
17008            fs::read_dir(root.join("subc-modules"))
17009                .unwrap()
17010                .map(|entry| entry.unwrap().file_type().unwrap())
17011                .filter(|kind| kind.is_dir())
17012                .count()
17013        };
17014        let supervisor =
17015            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17016                .with_cgroup_placement(Some(placement.clone()));
17017        let runtime = supervisor.runtime_config();
17018        let mut spec = ModuleSpec {
17019            module_id: "unique-spawn".into(),
17020            program: PathBuf::from("/bin/sleep"),
17021            args: vec!["60".into()],
17022            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17023                .into_iter()
17024                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17025                .collect(),
17026            reserved: false,
17027            reserved_prefixes: vec![],
17028            protocol: ModuleProtocol::None,
17029            overlap: Default::default(),
17030            carry_containment: false,
17031        };
17032        let spawn = |spec: &ModuleSpec| {
17033            spawn_child(
17034                spec,
17035                None,
17036                None,
17037                &runtime.stderr_ring,
17038                None,
17039                &runtime.child_roster,
17040                Some(&placement),
17041            )
17042            .unwrap()
17043        };
17044        let mut live = spawn(&spec);
17045        for _ in 0..3 {
17046            // A new process can enter the old slot while retirement is pending.
17047            let next = spawn(&spec);
17048            assert_ne!(live.module_id, next.module_id);
17049            live.start_kill().unwrap();
17050            live.wait().await.unwrap();
17051            live = next;
17052            assert!(
17053                live.child.try_wait().unwrap().is_none(),
17054                "retiring the old slot must not kill the replacement"
17055            );
17056            assert_eq!(
17057                group_count(),
17058                1,
17059                "only the live spawn's cgroup should remain"
17060            );
17061        }
17062        supervisor.begin_daemon_shutdown();
17063        let reap = tokio::spawn(async move {
17064            live.wait().await.unwrap();
17065        });
17066        supervisor
17067            .end_children_for_daemon_shutdown(false, std::future::pending())
17068            .await;
17069        reap.await.unwrap();
17070        assert_eq!(group_count(), 0);
17071        // A normal exit uses the same tree-cleanup path as a killed spawn.
17072        spec.program = PathBuf::from("/bin/true");
17073        spec.args.clear();
17074        let fresh_roster = ChildRoster::default();
17075        let mut short = spawn_child(
17076            &spec,
17077            None,
17078            None,
17079            &runtime.stderr_ring,
17080            None,
17081            &fresh_roster,
17082            Some(&placement),
17083        )
17084        .unwrap();
17085        short.wait().await.unwrap();
17086        assert_eq!(group_count(), 0);
17087        spec.module_id = "_".repeat(255);
17088        let mut long_id = spawn_child(
17089            &spec,
17090            None,
17091            None,
17092            &runtime.stderr_ring,
17093            None,
17094            &fresh_roster,
17095            Some(&placement),
17096        )
17097        .unwrap();
17098        long_id.wait().await.unwrap();
17099        assert_eq!(
17100            group_count(),
17101            0,
17102            "valid long module IDs must not exceed cgroup NAME_MAX"
17103        );
17104        fs::remove_dir(root.join("subc-modules")).unwrap();
17105        fs::remove_dir(root).unwrap();
17106        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17107    }
17108
17109    #[test]
17110    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17111        let path = Path::new("/definitely-missing-subc-cgroup");
17112        let mut command = Command::new("true");
17113        let error = apply_cgroup_placement(
17114            &mut command,
17115            &ModuleSpec {
17116                module_id: "broken-cgroup".to_string(),
17117                program: PathBuf::from("true"),
17118                args: Vec::new(),
17119                env: Vec::new(),
17120                reserved: false,
17121                reserved_prefixes: Vec::new(),
17122                protocol: ModuleProtocol::Subc,
17123                overlap: Default::default(),
17124                carry_containment: false,
17125            },
17126            path,
17127        )
17128        .expect_err("a parent cgroup open failure must reject the supervised spawn");
17129        let reason = error.to_string();
17130
17131        assert!(
17132            matches!(error, SuperviseError::Cgroup { .. }),
17133            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17134        );
17135        assert!(
17136            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17137            "parent cgroup open failure must name cgroup.procs: {reason}"
17138        );
17139    }
17140
17141    #[tokio::test]
17142    async fn reaping_a_child_removes_its_empty_module_cgroup() {
17143        let root = ScratchDir::new("supervisor-reap-cgroup");
17144        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17145        let placement = subc_cgroup::prepare_at(&root)
17146            .expect("prepare scratch cgroup root")
17147            .expect("scratch root has a cgroup.procs marker");
17148        let module_id = "reaped-module";
17149        let module = placement
17150            .module_path(module_id)
17151            .expect("create scratch module cgroup");
17152        let child = Command::new("true")
17153            .env("XDG_DATA_HOME", root.path())
17154            .env("XDG_RUNTIME_DIR", root.path())
17155            .env("XDG_CONFIG_HOME", root.path())
17156            .spawn()
17157            .expect("spawn short-lived child");
17158        let pid = child.id().expect("spawned child has pid");
17159        let mut child = SupervisedChild {
17160            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17161            child: crate::supervise::containment::ProcessChild::plain(child, pid),
17162            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17163            protocol: ModuleProtocol::Subc,
17164            module_id: module_id.to_string(),
17165            cgroup_placement: Some(placement),
17166            stdout_pump: None,
17167            stderr_pump: None,
17168            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17169            spawned_at: tokio::time::Instant::now(),
17170            registration_deadline: None,
17171            registration_watch: None,
17172            spawned_at_ms: 0,
17173            spawned_from: PathBuf::from("true"),
17174            launch_spec_digest: LaunchSpecDigest([0; 32]),
17175            spawned_file_identity: None,
17176            process_start_time: None,
17177            process_identity: None,
17178            pid,
17179            roster_guard: None,
17180            #[cfg(target_os = "macos")]
17181            privacy_exec: None,
17182            spawn_failure: None,
17183        };
17184
17185        child.wait().await.expect("reap short-lived child");
17186
17187        assert!(
17188            !module.exists(),
17189            "waiting for a non-carried child must remove its empty cgroup before returning"
17190        );
17191        child.finish_exit(false).await.unwrap();
17192    }
17193
17194    #[test]
17195    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17196        let root = ScratchDir::new("supervisor-non-empty-cgroup");
17197        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17198        let placement = subc_cgroup::prepare_at(&root)
17199            .expect("prepare scratch cgroup root")
17200            .expect("scratch root has a cgroup.procs marker");
17201        let module = placement
17202            .module_path("surviving-module")
17203            .expect("create scratch module cgroup");
17204        fs::write(module.join("surviving-process"), b"still present")
17205            .expect("make scratch cgroup non-empty");
17206        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17207
17208        remove_module_cgroup(&placement, "surviving-module");
17209
17210        let logs = crate::router::test_log::captured_logs(&logs);
17211        assert!(
17212            module.exists(),
17213            "failed removal must leave the cgroup intact"
17214        );
17215        assert!(
17216            logs.contains("could not remove module cgroup after process exit; continuing teardown")
17217                && logs.contains("surviving-module"),
17218            "best-effort removal must report the failure without returning it: {logs}"
17219        );
17220    }
17221
17222    #[test]
17223    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17224        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17225        let reason = SuperviseError::Spawn {
17226            program: PathBuf::from("/bin/true"),
17227            source: io::Error::from_raw_os_error(13),
17228            cgroup_path: Some(cgroup_path.clone()),
17229        }
17230        .to_string();
17231
17232        assert!(
17233            reason.contains(&cgroup_path.display().to_string()),
17234            "a pre_exec spawn failure must name the cgroup path: {reason}"
17235        );
17236    }
17237}
17238
17239#[cfg(test)]
17240mod spawn_subscriber_lag_tests {
17241    use super::*;
17242
17243    /// A subscriber whose connection stops draining is dropped once its frame
17244    /// channel fills. The client must learn that from a terminal Error frame
17245    /// after the frames already queued for it, not from a stream that simply
17246    /// goes quiet.
17247    #[tokio::test]
17248    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17249        let feed = SpawnEventFeed::default();
17250        feed.configure_incarnation("lag-incarnation".to_string());
17251        // A one-slot connection queue that nobody reads until the emits are
17252        // done: the forwarder parks on it and the subscriber channel fills.
17253        let (tx, mut rx) = mpsc::channel(1);
17254        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17255            .expect("subscribe");
17256        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17257        for index in 0..emitted {
17258            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17259            // Let the forwarder take what it can so the fill point is the
17260            // subscriber channel, not a scheduling accident.
17261            tokio::task::yield_now().await;
17262        }
17263        assert_eq!(
17264            feed.subscriber_count(),
17265            0,
17266            "the lagged subscriber must be removed"
17267        );
17268
17269        let mut data = Vec::new();
17270        let mut last = None;
17271        loop {
17272            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17273                .await
17274                .expect("the forwarder must finish once the subscriber is dropped");
17275            let Some(outbound) = next else { break };
17276            let frame = outbound.frame;
17277            if frame.header.ty == FrameType::StreamData {
17278                assert!(last.is_none(), "no data may follow the terminal frame");
17279                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17280                data.push(event.cursor.seq);
17281            } else {
17282                assert!(last.is_none(), "exactly one terminal frame");
17283                last = Some(frame);
17284            }
17285        }
17286        assert!(!data.is_empty(), "queued frames drain before the terminal");
17287        for pair in data.windows(2) {
17288            assert_eq!(
17289                pair[1],
17290                pair[0] + 1,
17291                "queued frames arrive dense and in order"
17292            );
17293        }
17294        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17295        assert_eq!(terminal.header.ty, FrameType::Error);
17296        assert_eq!(terminal.header.corr, 7);
17297        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17298        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17299        let detail = body.detail.expect("lagged error carries detail");
17300        assert_eq!(
17301            detail["first_undelivered_cursor"]["seq"],
17302            data.last().unwrap() + 1,
17303            "the named cursor is the first event the subscriber did not receive"
17304        );
17305        assert_eq!(
17306            detail["first_undelivered_cursor"]["daemon_incarnation"],
17307            "lag-incarnation"
17308        );
17309    }
17310}
17311
17312#[cfg(test)]
17313mod terminal_history_read_concurrency_tests {
17314    use super::*;
17315    use crate::terminal_journal::read_pause;
17316    use cortexkit_test_support::ScratchDir;
17317    use std::sync::mpsc as std_mpsc;
17318
17319    fn journaled_ring(
17320        journal: &Arc<crate::terminal_journal::TerminalJournal>,
17321    ) -> Arc<Mutex<TerminalRing>> {
17322        Arc::new(Mutex::new(
17323            TerminalRing::new(TerminalRingConfig::default(), 1)
17324                .with_journal(Some(Arc::clone(journal))),
17325        ))
17326    }
17327
17328    fn crash(at_ms: u64) -> ExitReport {
17329        ExitReport {
17330            kind: ExitKind::Crash,
17331            code: Some(1),
17332            signal: None,
17333            at_ms,
17334        }
17335    }
17336
17337    /// Record an exit on another thread and report whether it finished within
17338    /// `bound`. The recorder thread is left running if it did not.
17339    fn record_within(
17340        module_id: &'static str,
17341        ring: &Arc<Mutex<TerminalRing>>,
17342        at_ms: u64,
17343        bound: Duration,
17344    ) -> bool {
17345        let ring = Arc::clone(ring);
17346        let (done, done_rx) = std_mpsc::channel();
17347        std::thread::spawn(move || {
17348            record_terminal(
17349                module_id,
17350                &ring,
17351                &SpawnEventFeed::default(),
17352                &crash(at_ms),
17353                TerminalDisposition::Restarting,
17354            );
17355            let _ = done.send(());
17356        });
17357        done_rx.recv_timeout(bound).is_ok()
17358    }
17359
17360    /// A history read in progress must not hold the journal writer (which every
17361    /// module's exit recording needs) or the module's own ring. Exits recorded
17362    /// while the read is paused complete promptly; the paused read answers as of
17363    /// the moment it started, and the next read has each exit exactly once.
17364    #[test]
17365    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17366        let dir = ScratchDir::new("terminal-history-concurrent-read");
17367        let path = dir.join("terminals.jsonl");
17368        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17369            path.clone(),
17370            "daemon".into(),
17371        ));
17372        let reader_ring = journaled_ring(&journal);
17373        let other_ring = journaled_ring(&journal);
17374        assert!(record_within(
17375            "reader-module",
17376            &reader_ring,
17377            10,
17378            Duration::from_secs(5)
17379        ));
17380
17381        let (started, release) = read_pause::install(&path);
17382        let reading = {
17383            let ring = Arc::clone(&reader_ring);
17384            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17385        };
17386        started
17387            .recv_timeout(Duration::from_secs(5))
17388            .expect("the history read reached its pause");
17389
17390        let bound = Duration::from_secs(1);
17391        assert!(
17392            record_within("other-module", &other_ring, 20, bound),
17393            "another module's exit waited on a history read (journal writer held)"
17394        );
17395        assert!(
17396            record_within("reader-module", &reader_ring, 30, bound),
17397            "the read module's own exit waited on its history read (ring held)"
17398        );
17399
17400        drop(release);
17401        let paused = reading.join().unwrap();
17402        assert_eq!(
17403            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17404            vec![10],
17405            "an exit recorded after the read began lands in neither half of it"
17406        );
17407        assert_eq!(paused.journal_skipped_lines, 0);
17408        assert_eq!(paused.journal_read_errors, 0);
17409
17410        let after = durable_terminal_history_of(&reader_ring, "reader-module");
17411        assert_eq!(
17412            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17413            vec![10, 30],
17414            "the next read merges ring and journal with no duplicate"
17415        );
17416        assert_eq!(after.journal_skipped_lines, 0);
17417    }
17418}
17419
17420/// What a restart does with the exited process's output readers. These drive
17421/// the same settle functions the supervisor calls, with a reader the test
17422/// holds, so a reader that has not been scheduled by the bound is a controlled
17423/// input rather than something only a loaded machine produces.
17424#[cfg(test)]
17425mod stderr_settle_tests {
17426    use std::{
17427        future::Future,
17428        io,
17429        pin::Pin,
17430        sync::{Arc, Mutex},
17431        task::{Context, Poll},
17432        time::Duration,
17433    };
17434
17435    use tokio::{
17436        io::{AsyncRead, ReadBuf},
17437        sync::oneshot,
17438        time::Instant,
17439    };
17440
17441    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17442    use crate::stderr_tail::{
17443        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17444        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17445    };
17446
17447    const BOUND: Duration = Duration::from_millis(250);
17448
17449    /// Yields `before`, then stays pending until the gate is released, then
17450    /// yields `after` and reaches EOF. The bytes after the gate were written
17451    /// by a process that has already exited; only the reader is behind.
17452    struct HeldReader {
17453        before: Option<Vec<u8>>,
17454        gate: Option<oneshot::Receiver<()>>,
17455        after: io::Cursor<Vec<u8>>,
17456    }
17457
17458    impl AsyncRead for HeldReader {
17459        fn poll_read(
17460            mut self: Pin<&mut Self>,
17461            cx: &mut Context<'_>,
17462            buf: &mut ReadBuf<'_>,
17463        ) -> Poll<io::Result<()>> {
17464            if let Some(bytes) = self.before.take() {
17465                buf.put_slice(&bytes);
17466                return Poll::Ready(Ok(()));
17467            }
17468            if let Some(gate) = self.gate.as_mut() {
17469                match Pin::new(gate).poll(cx) {
17470                    Poll::Pending => return Poll::Pending,
17471                    Poll::Ready(_) => self.gate = None,
17472                }
17473            }
17474            Pin::new(&mut self.after).poll_read(cx, buf)
17475        }
17476    }
17477
17478    struct DiscardSink;
17479
17480    impl OutputSink for DiscardSink {
17481        fn write_line(&mut self, _line: &[u8]) {}
17482    }
17483
17484    fn line(text: &str) -> TailEntry {
17485        TailEntry::Line {
17486            text: text.to_string(),
17487            truncated: false,
17488            at_ms: None,
17489        }
17490    }
17491
17492    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17493        ring.lock().unwrap()
17494    }
17495
17496    /// Start a reader for a new process generation that delivers `before`
17497    /// immediately and `after` only once the returned sender fires (or is
17498    /// dropped).
17499    fn held_pump(
17500        ring: &Arc<Mutex<StderrRing>>,
17501        before: &str,
17502        after: &str,
17503    ) -> (StderrPump, oneshot::Sender<()>) {
17504        let generation = lock(ring).begin_process();
17505        let (release, gate) = oneshot::channel();
17506        let reader = HeldReader {
17507            before: Some(before.as_bytes().to_vec()),
17508            gate: Some(gate),
17509            after: io::Cursor::new(after.as_bytes().to_vec()),
17510        };
17511        let task = tokio::spawn(pump_stderr_to(
17512            reader,
17513            Arc::clone(ring),
17514            generation,
17515            DiscardSink,
17516        ));
17517        (StderrPump { task, generation }, release)
17518    }
17519
17520    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17521        for _ in 0..1000 {
17522            if done(&lock(ring)) {
17523                return;
17524            }
17525            tokio::time::sleep(Duration::from_millis(1)).await;
17526        }
17527        panic!(
17528            "ring never reached the expected state: {:?}",
17529            lock(ring).snapshot(None, None)
17530        );
17531    }
17532
17533    #[tokio::test(start_paused = true)]
17534    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17535        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17536        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17537
17538        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17539        let before_release = lock(&ring).snapshot(None, None);
17540        assert!(
17541            matches!(before_release.capture, CaptureState::Incomplete { .. }),
17542            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17543        );
17544
17545        // The restart: the next process starts and writes before the old
17546        // reader catches up.
17547        let next = lock(&ring).begin_process();
17548        lock(&ring).push_line_from(next, "next process booting");
17549        release.send(()).unwrap();
17550        wait_until(&ring, |ring| {
17551            ring.snapshot(None, None).capture == CaptureState::Captured
17552        })
17553        .await;
17554
17555        assert_eq!(
17556            untimed(lock(&ring).snapshot(None, None).entries),
17557            vec![
17558                line("booting"),
17559                line("config error: missing storage"),
17560                TailEntry::ProcessStart,
17561                line("next process booting"),
17562            ],
17563            "the crash's last line must survive a slow reader and stay in the crashed process's section"
17564        );
17565    }
17566
17567    #[tokio::test(start_paused = true)]
17568    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17569    ) {
17570        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17571        // `_held` is never fired: a descendant keeps the pipe open for the
17572        // whole test.
17573        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17574
17575        let started = Instant::now();
17576        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17577        assert_eq!(
17578            started.elapsed(),
17579            BOUND,
17580            "the restart must wait exactly the bound for a pipe that stays open, no longer"
17581        );
17582
17583        let next = lock(&ring).begin_process();
17584        lock(&ring).push_line_from(next, "next process booting");
17585        tokio::time::sleep(Duration::from_secs(60)).await;
17586
17587        let snapshot = lock(&ring).snapshot(None, None);
17588        match &snapshot.capture {
17589            CaptureState::Incomplete { reason } => assert!(
17590                reason.contains("had not reached EOF") && reason.contains("250ms"),
17591                "the reason must say what is missing and after how long: {reason}"
17592            ),
17593            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17594        }
17595        assert_eq!(
17596            untimed(snapshot.entries),
17597            vec![
17598                line("parent exiting"),
17599                TailEntry::ProcessStart,
17600                line("next process booting"),
17601            ]
17602        );
17603    }
17604
17605    #[tokio::test(start_paused = true)]
17606    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17607        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17608        let (pump, release) = held_pump(&ring, "one\n", "two\n");
17609        release.send(()).unwrap();
17610
17611        settle_stderr_pump("clean", &ring, pump, BOUND).await;
17612
17613        let snapshot = lock(&ring).snapshot(None, None);
17614        assert_eq!(snapshot.capture, CaptureState::Captured);
17615        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17616    }
17617
17618    #[tokio::test(start_paused = true)]
17619    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17620        const BURST: usize = 150;
17621        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17622        let path = capture.join("two-pipe.stderr.log");
17623        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17624        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17625        let generation = lock(&ring).begin_process();
17626        let stdout = (0..BURST)
17627            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17628            .collect::<String>();
17629        let stderr = (0..BURST)
17630            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17631            .collect::<String>();
17632        // Model a child that has written both bursts and exited. Hold the stdout
17633        // reader after one line and a partial prefix, as if pipe readiness or
17634        // capture-file writes had delayed it. The final line also needs EOF to
17635        // flush: neither complete nor partial buffered output may be discarded.
17636        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17637        let (release, gate) = oneshot::channel();
17638        let reader = HeldReader {
17639            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17640            gate: Some(gate),
17641            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17642        };
17643        let (finished, completion) = oneshot::channel();
17644        let stdout_sink = sink.clone();
17645        let pump = tokio::spawn(async move {
17646            pump_stdout_to(reader, stdout_sink).await;
17647            finished.send(()).unwrap();
17648        });
17649        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17650
17651        let started = Instant::now();
17652        settle_stdout_pump("two-pipe", pump, BOUND).await;
17653        assert_eq!(
17654            started.elapsed(),
17655            BOUND,
17656            "a late reader must not hold up restart"
17657        );
17658        assert_eq!(
17659            std::fs::read_to_string(&path).unwrap().lines().count(),
17660            BURST + 1
17661        );
17662
17663        release
17664            .send(())
17665            .expect("stdout reader must survive the exit drain bound");
17666        tokio::time::timeout(Duration::from_secs(1), completion)
17667            .await
17668            .expect("released stdout reader must reach EOF")
17669            .expect("stdout capture must finish at EOF, not be cancelled");
17670        let contents = std::fs::read_to_string(&path).unwrap();
17671        let mut lines = contents
17672            .lines()
17673            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17674            .collect::<Vec<_>>();
17675        let mut expected = (0..BURST)
17676            .flat_map(|i| {
17677                [
17678                    format!("out-{i:04}-{}", "o".repeat(64)),
17679                    format!("err-{i:04}-{}", "e".repeat(64)),
17680                ]
17681            })
17682            .collect::<Vec<_>>();
17683        lines.sort_unstable();
17684        expected.sort_unstable();
17685        assert_eq!(
17686            lines, expected,
17687            "both pipes must retain every complete line exactly once"
17688        );
17689    }
17690}
17691
17692/// Containment of a module's process tree (issue #109).
17693///
17694/// The behaviour these defend against is a module helper surviving its module:
17695/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
17696/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
17697/// compounds it.
17698///
17699/// They run against the SUPERVISOR rather than the job-object crate because the
17700/// claim is about teardown: a crate-level test proves a job can reap a tree, not
17701/// that the daemon's drain path reaches it.
17702///
17703/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
17704/// lane there is a separate containment path with its own tests.
17705#[cfg(all(test, windows))]
17706mod job_containment_tests {
17707    use super::*;
17708    use cortexkit_test_support::ScratchDir;
17709    use std::{
17710        path::{Path, PathBuf},
17711        sync::{Arc, Mutex},
17712        time::{Duration, Instant},
17713    };
17714
17715    /// The stub, expected beside this test executable.
17716    ///
17717    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17718    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17719    /// failure then reads as a broken test rather than an unbuilt dependency.
17720    fn stub_path() -> PathBuf {
17721        let mut path = std::env::current_exe().expect("current_exe available in tests");
17722        path.pop();
17723        path.pop();
17724        path.push("fake-aft-stub.exe");
17725        assert!(
17726            path.exists(),
17727            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17728             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17729            path.display()
17730        );
17731        path
17732    }
17733
17734    /// Poll for the grandchild pid the stub records, and parse it.
17735    fn read_grandchild_pid(path: &Path) -> u32 {
17736        let deadline = Instant::now() + Duration::from_secs(10);
17737        loop {
17738            if let Ok(contents) = std::fs::read_to_string(path) {
17739                if let Ok(pid) = contents.trim().parse() {
17740                    return pid;
17741                }
17742            }
17743            assert!(
17744                Instant::now() < deadline,
17745                "the stub never recorded a grandchild pid at {}",
17746                path.display()
17747            );
17748            std::thread::sleep(Duration::from_millis(10));
17749        }
17750    }
17751
17752    /// Everything one fixture run needs, so the two tests below differ in exactly
17753    /// one place: whether the child is contained.
17754    struct Fixture {
17755        _dir: ScratchDir,
17756        module_id: String,
17757        grandchild: u32,
17758        child: Option<SupervisedChild>,
17759        registry: Arc<Registry>,
17760        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17761        terminal_ring: Arc<Mutex<TerminalRing>>,
17762        spawn_events: SpawnEventFeed,
17763    }
17764
17765    fn fixture(label: &str, module_id: &str) -> Fixture {
17766        let dir = ScratchDir::new(label);
17767        let pid_file = dir.join("grandchild.pid");
17768        let supervisor = Supervisor::new_for_test(
17769            Arc::new(Registry::default()),
17770            RestartPolicy::new(3, Duration::ZERO),
17771        );
17772        let runtime = supervisor.runtime_config();
17773        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17774        let spec = ModuleSpec {
17775            module_id: module_id.to_string(),
17776            program: stub_path(),
17777            // Zero args deliberately: a `--subc` argument would make the stub dial
17778            // a daemon that is not there, and the failure would land in the same
17779            // stderr ring this fixture exists to keep quiet.
17780            args: Vec::new(),
17781            env: vec![
17782                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17783                (
17784                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17785                    pid_file.display().to_string(),
17786                ),
17787            ],
17788            reserved: false,
17789            reserved_prefixes: Vec::new(),
17790            protocol: ModuleProtocol::Subc,
17791            overlap: Default::default(),
17792            carry_containment: false,
17793        };
17794        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17795            .expect("spawn the supervised fixture");
17796        let grandchild = read_grandchild_pid(&pid_file);
17797        Fixture {
17798            _dir: dir,
17799            module_id: module_id.to_string(),
17800            grandchild,
17801            child: Some(child),
17802            registry: Arc::new(Registry::default()),
17803            snapshot,
17804            terminal_ring: Arc::clone(&runtime.terminal_ring),
17805            spawn_events: SpawnEventFeed::default(),
17806        }
17807    }
17808
17809    impl Fixture {
17810        /// Drain through the supervisor's own teardown path.
17811        async fn drain(&mut self) {
17812            let child = self
17813                .child
17814                .take()
17815                .expect("the fixture child is still present");
17816            drain_child_to_state(
17817                &self.module_id,
17818                ModuleProtocol::Subc,
17819                // No forwarding table in this fixture, so nothing reaches the
17820                // child over a connection.
17821                StopNotice::NotSent,
17822                &self.registry,
17823                None,
17824                &self.snapshot,
17825                &self.terminal_ring,
17826                &self.spawn_events,
17827                child,
17828                Duration::from_millis(500),
17829                ModuleState::Stopped,
17830                Some(false),
17831                &FailureContext::operator(),
17832            )
17833            .await
17834            .expect("drain the supervised fixture");
17835        }
17836    }
17837
17838    /// Teardown reaps the grandchild, not merely the direct child.
17839    ///
17840    /// This is the assertion the change exists for. Before containment the
17841    /// grandchild survived: it is a separate process, and `start_kill` is
17842    /// `TerminateProcess` scoped to one pid.
17843    #[tokio::test]
17844    async fn teardown_reaps_the_grandchild() {
17845        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17846        let grandchild = fixture.grandchild;
17847
17848        assert!(
17849            subc_jobobject::process_exists(grandchild),
17850            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17851        );
17852
17853        fixture.drain().await;
17854
17855        assert!(
17856            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17857            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17858        );
17859    }
17860
17861    /// The mutation control: with containment withheld, the grandchild survives
17862    /// the same kill.
17863    ///
17864    /// This is the defect reproduction from #109 — a direct-child kill reaches
17865    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17866    /// supervisor because `spawn_and_mark_running` now always contains on
17867    /// Windows, which is the point: there is no longer a path that spawns
17868    /// uncontained, so the control has to construct one.
17869    ///
17870    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17871    /// grandchild ever dies here, that test is passing for a reason unrelated to
17872    /// the job object and the containment claim is unproven.
17873    #[test]
17874    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17875        let dir = ScratchDir::new("teardown-uncontained");
17876        let pid_file = dir.join("grandchild.pid");
17877        let mut child = std::process::Command::new(stub_path())
17878            .env("FAKE_AFT_NEVER_CONNECT", "1")
17879            .env(
17880                "FAKE_AFT_GRANDCHILD_PID_FILE",
17881                pid_file.display().to_string(),
17882            )
17883            .stdin(std::process::Stdio::null())
17884            .stdout(std::process::Stdio::null())
17885            .stderr(std::process::Stdio::null())
17886            .spawn()
17887            .expect("spawn the uncontained fixture");
17888        let grandchild = read_grandchild_pid(&pid_file);
17889
17890        // Exactly what the pre-fix teardown did: kill the direct child.
17891        child.kill().expect("kill the direct child");
17892        let _ = child.wait();
17893
17894        assert!(
17895            subc_jobobject::process_exists(grandchild),
17896            "grandchild {grandchild} died with the direct child, so this control no longer \
17897             distinguishes contained from uncontained teardown and the regression test is \
17898             passing vacuously"
17899        );
17900
17901        // The orphan this control demonstrates is the leak the fix prevents, so
17902        // the control must not leave one behind.
17903        kill_tree(grandchild);
17904    }
17905
17906    /// Crash durability: closing the containment handle reaps the tree with no
17907    /// teardown code running at all.
17908    ///
17909    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17910    /// call anything — and it is why containment is a kernel property of the
17911    /// handle rather than a step in the drain. Discovered by getting the
17912    /// mutation control wrong: clearing `job` to "disable" containment instead
17913    /// killed the tree, which is the guarantee, not a mistake.
17914    #[tokio::test]
17915    async fn dropping_containment_reaps_the_grandchild() {
17916        let mut fixture = fixture("drop-containment", "tree-drop");
17917        let grandchild = fixture.grandchild;
17918
17919        assert!(subc_jobobject::process_exists(grandchild));
17920
17921        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17922        fixture.child.as_mut().expect("child present").job = None;
17923
17924        assert!(
17925            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17926            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17927             crash would leave the tree behind"
17928        );
17929    }
17930
17931    /// Kill a pid and its tree, then confirm it is gone.
17932    fn kill_tree(pid: u32) {
17933        let _ = std::process::Command::new("taskkill.exe")
17934            .args(["/PID", &pid.to_string(), "/T", "/F"])
17935            .stdin(std::process::Stdio::null())
17936            .stdout(std::process::Stdio::null())
17937            .stderr(std::process::Stdio::null())
17938            .status();
17939        assert!(
17940            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17941            "could not clean up grandchild {pid}"
17942        );
17943    }
17944}
17945
17946#[cfg(test)]
17947mod privacy_trampoline_configuration_tests {
17948    #[tokio::test]
17949    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17950    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17951        #[cfg(target_os = "macos")]
17952        {
17953            let supervisor = super::Supervisor::new(
17954                std::sync::Arc::new(crate::Registry::default()),
17955                super::RestartPolicy::default(),
17956            );
17957            let error = supervisor.spawn(spec()).unwrap_err();
17958            assert!(
17959                error
17960                    .to_string()
17961                    .contains("no privacy trampoline configured"),
17962                "{error}"
17963            );
17964        }
17965    }
17966
17967    #[tokio::test]
17968    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17969    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17970        #[cfg(target_os = "macos")]
17971        {
17972            let supervisor = super::Supervisor::new(
17973                std::sync::Arc::new(crate::Registry::default()),
17974                super::RestartPolicy::default(),
17975            )
17976            .with_privacy_trampoline(std::env::current_exe().unwrap());
17977            let error = supervisor.spawn(spec()).unwrap_err();
17978            assert!(
17979                error
17980                    .to_string()
17981                    .contains("binary does not implement the privacy trampoline protocol"),
17982                "{error}"
17983            );
17984        }
17985    }
17986
17987    #[cfg(target_os = "macos")]
17988    fn spec() -> super::ModuleSpec {
17989        super::ModuleSpec {
17990            module_id: "privacy-configuration".into(),
17991            program: "/bin/sleep".into(),
17992            args: vec!["30".into()],
17993            env: vec![],
17994            reserved: false,
17995            reserved_prefixes: vec![],
17996            protocol: subc_control::ModuleProtocol::None,
17997            overlap: super::ModuleOverlap::Exclusive,
17998            carry_containment: false,
17999        }
18000    }
18001}
18002
18003#[cfg(test)]
18004mod privacy_exec_boundary_tests {
18005    #[cfg(target_os = "macos")]
18006    use super::*;
18007    #[cfg(target_os = "macos")]
18008    use std::{
18009        io::{Read, Write},
18010        net::{TcpListener, TcpStream},
18011    };
18012
18013    /// Unit-test-only pause at the actual early image read, not at a later
18014    /// status read. Production supervisors never inspect this environment key.
18015    #[cfg(target_os = "macos")]
18016    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18017        if let Some((_, path)) = spec
18018            .env
18019            .iter()
18020            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18021        {
18022            let mut barrier = TcpStream::connect(path).unwrap();
18023            barrier
18024                .set_read_timeout(Some(Duration::from_secs(30)))
18025                .unwrap();
18026            barrier.write_all(&pid.to_ne_bytes()).unwrap();
18027            let mut release = [0];
18028            barrier.read_exact(&mut release).unwrap();
18029            assert_eq!(&release, b"X");
18030        }
18031    }
18032
18033    #[cfg(target_os = "macos")]
18034    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18035        ModuleSpec {
18036            module_id: "privacy-boundary".into(),
18037            program: program.into(),
18038            args: args.iter().map(|arg| (*arg).into()).collect(),
18039            env: vec![],
18040            reserved: false,
18041            reserved_prefixes: vec![],
18042            protocol: ModuleProtocol::None,
18043            overlap: ModuleOverlap::Exclusive,
18044            carry_containment: false,
18045        }
18046    }
18047
18048    #[cfg(target_os = "macos")]
18049    async fn accept(listener: TcpListener) -> TcpStream {
18050        // Socket readiness, not elapsed time, establishes both pause points.
18051        let listener = tokio::net::TcpListener::from_std({
18052            listener.set_nonblocking(true).unwrap();
18053            listener
18054        })
18055        .unwrap();
18056        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18057            .await
18058            .unwrap()
18059            .unwrap();
18060        let stream = stream.into_std().unwrap();
18061        stream.set_nonblocking(false).unwrap();
18062        stream
18063            .set_read_timeout(Some(Duration::from_secs(30)))
18064            .unwrap();
18065        stream
18066    }
18067
18068    #[tokio::test]
18069    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18070    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18071        #[cfg(target_os = "macos")]
18072        {
18073            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18074            // Loopback sockets also work when the replay adapter's TMPDIR is
18075            // longer than Darwin's Unix-domain socket path limit.
18076            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18077            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18078            let record = root.join("live-children.json");
18079            let supervisor =
18080                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18081                    .with_live_children_record(&record);
18082            let runtime = supervisor.runtime_config();
18083            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18084            let mut spec = spec("/bin/sleep", &["30"]);
18085            spec.env = vec![
18086                (
18087                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18088                    exec_listener.local_addr().unwrap().to_string(),
18089                ),
18090                (
18091                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18092                    sample_listener.local_addr().unwrap().to_string(),
18093                ),
18094            ];
18095            let spawn = tokio::task::spawn_blocking(move || {
18096                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18097            });
18098            let mut sample = accept(sample_listener).await;
18099            let mut pid = [0; 4];
18100            sample.read_exact(&mut pid).unwrap();
18101            let pid = u32::from_ne_bytes(pid);
18102            let mut exec = accept(exec_listener).await;
18103            let mut ready = [0];
18104            exec.read_exact(&mut ready).unwrap();
18105            assert_eq!(&ready, b"R");
18106            // The early read is guaranteed to see a real, non-null trampoline
18107            // image: the fixture has reached its barrier and cannot exec yet.
18108            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18109            assert_eq!(
18110                observe_spawned_image(pid).unwrap().executable,
18111                Some(trampoline)
18112            );
18113            sample.write_all(b"X").unwrap();
18114            let mut child = spawn.await.unwrap();
18115            let early = crate::live_children::read_record(&record).unwrap();
18116            assert_eq!(early.len(), 1);
18117            assert_eq!(early[0].pid, pid);
18118            assert_eq!(
18119                early[0].executable, None,
18120                "unconfirmed trampoline image entered the roster"
18121            );
18122            assert!(child.report_ready.get().is_none());
18123            // The barrier's duration is unrelated to the production five-second
18124            // exec budget. Start the test's confirmation budget upon release.
18125            child.privacy_exec.as_mut().unwrap().deadline =
18126                tokio::time::Instant::now() + Duration::from_secs(30);
18127            exec.write_all(b"X").unwrap();
18128            child.confirm_privacy_exec().await;
18129            assert_eq!(child.spawn_failure, None);
18130            assert!(child.report_ready.get().is_some());
18131            let confirmed = crate::live_children::read_record(&record).unwrap();
18132            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18133            assert_ne!(module, trampoline);
18134            assert_eq!(confirmed[0].executable, Some(module.into()));
18135            child.start_kill().unwrap();
18136            child.wait().await.unwrap();
18137            child.release_roster();
18138        }
18139    }
18140
18141    #[tokio::test]
18142    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18143    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18144        #[cfg(target_os = "macos")]
18145        {
18146            let registry = Arc::new(Registry::default());
18147            let policy = RestartPolicy::new(0, Duration::ZERO);
18148            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
18149            let runtime = supervisor.runtime_config();
18150            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18151            let spec = spec("/bin/sh", &["-c", "exit 121"]);
18152            // Drive spawn and confirmation separately instead of starting the
18153            // monitor. WNOWAIT observes a real exit without consuming its status,
18154            // so confirmation's first try_wait must take the already-exited arm.
18155            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18156            let pid = child.pid;
18157            tokio::task::spawn_blocking(move || {
18158                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18159            })
18160            .await
18161            .unwrap()
18162            .unwrap();
18163            child.privacy_exec.as_mut().unwrap().deadline =
18164                tokio::time::Instant::now() + Duration::from_secs(30);
18165            let status = child.wait().await.unwrap();
18166            assert_eq!(status.code(), Some(121));
18167            assert!(child.privacy_exec.is_none());
18168            assert!(
18169                child.report_ready.get().is_none(),
18170                "an exited module must not publish a live pid"
18171            );
18172            let report = classify_reaped_child_exit(&snapshot, &child, &status);
18173            on_child_exit(
18174                &spec,
18175                policy,
18176                &registry,
18177                &snapshot,
18178                &runtime.terminal_ring,
18179                &runtime.spawn_events,
18180                &runtime.child_roster,
18181                report,
18182                true,
18183            )
18184            .await;
18185            let state = lock_snapshot(&snapshot).unwrap();
18186            assert_eq!(state.state, ModuleState::Failed);
18187            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18188            assert_eq!(state.reported_pid(), None);
18189            drop(state);
18190            let history = runtime.terminal_ring.lock().unwrap().snapshot();
18191            assert_eq!(history.entries.len(), 1);
18192            let terminal = &history.entries[0];
18193            assert_eq!(terminal.exit_code, Some(121));
18194            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18195            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18196            assert_eq!(
18197                terminal.disposition_detail.as_deref(),
18198                Some(policy.budget_exhausted_detail().as_str()),
18199                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18200            );
18201            assert_eq!(child.spawn_failure, None);
18202            child.release_roster();
18203        }
18204    }
18205}
18206
18207/// The daemon's real spawn path hands a subc-wire child its launch nonce on
18208/// descriptor 3, without an environment copy. The shell records the nonce
18209/// and its environment after exec so these tests observe the real handover.
18210#[cfg(all(test, unix))]
18211mod launch_nonce_descriptor_tests {
18212    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18213    use crate::stderr_tail::{StderrRing, StderrTailConfig};
18214    use cortexkit_test_support::ScratchDir;
18215    use std::{
18216        path::PathBuf,
18217        sync::{Arc, Mutex},
18218        time::{Duration, Instant},
18219    };
18220
18221    async fn probe(role: super::SpawnRole) {
18222        let scratch = ScratchDir::new("launch-nonce-descriptor");
18223        let fd_copy = scratch.join("from-descriptor");
18224        let env_copy = scratch.join("environment");
18225        let script = format!(
18226            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18227            fd = fd_copy.display(), env = env_copy.display(),
18228        );
18229        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18230        let spec = ModuleSpec {
18231            module_id: "nonce-descriptor-probe".to_string(),
18232            program: PathBuf::from("/bin/sh"),
18233            args: vec!["-c".to_string(), script],
18234            env: vec![
18235                xdg("XDG_DATA_HOME"),
18236                xdg("XDG_RUNTIME_DIR"),
18237                xdg("XDG_CONFIG_HOME"),
18238                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18239            ],
18240            reserved: true,
18241            reserved_prefixes: Vec::new(),
18242            protocol: ModuleProtocol::Subc,
18243            overlap: Default::default(),
18244            carry_containment: false,
18245        };
18246        let handle = SupervisorHandle::new();
18247        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18248        let roster = ChildRoster::default();
18249        #[cfg(target_os = "macos")]
18250        {
18251            let path = super::test_privacy_trampoline();
18252            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18253        }
18254        let child = super::spawn_child_in_slot(
18255            &spec,
18256            None,
18257            Some(&handle),
18258            &ring,
18259            None,
18260            &roster,
18261            #[cfg(target_os = "linux")]
18262            None,
18263            role,
18264            matches!(role, super::SpawnRole::SwapCandidate),
18265        )
18266        .expect("spawn probe");
18267        let deadline = Instant::now() + Duration::from_secs(10);
18268        while !(fd_copy.exists() && env_copy.exists()) {
18269            assert!(Instant::now() < deadline, "probe never wrote its copies");
18270            tokio::time::sleep(Duration::from_millis(20)).await;
18271        }
18272        let nonce = std::fs::read_to_string(fd_copy).unwrap();
18273        assert!(!nonce.is_empty());
18274        let environment = std::fs::read_to_string(env_copy).unwrap();
18275        assert!(environment
18276            .lines()
18277            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18278        let copy = environment
18279            .lines()
18280            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18281        assert_eq!(
18282            copy, None,
18283            "Unix children must never receive the environment nonce"
18284        );
18285        if matches!(role, super::SpawnRole::Plain) {
18286            assert_eq!(
18287                handle.spawn_nonce(&spec.module_id).as_deref(),
18288                Some(nonce.as_str())
18289            );
18290        }
18291        drop(child);
18292    }
18293
18294    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18295    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18296        probe(super::SpawnRole::Plain).await;
18297    }
18298
18299    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18300    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18301        probe(super::SpawnRole::SwapCandidate).await;
18302    }
18303}
18304
18305#[cfg(all(test, target_os = "linux"))]
18306#[path = "supervise/cgroup_reap_tests.rs"]
18307mod cgroup_reap_tests;
18308
18309#[cfg(all(test, target_os = "linux"))]
18310mod cgroup_containment_tests {
18311    use super::*;
18312    use cortexkit_test_support::ScratchDir;
18313
18314    #[tokio::test]
18315    async fn linux_subtree_memory_covers_module_children() {
18316        let dir = ScratchDir::new("subtree-memory");
18317        let root = PathBuf::from(format!(
18318            "/sys/fs/cgroup/subc-memory-test-{}-{}",
18319            std::process::id(),
18320            unix_ms_now()
18321        ));
18322        if let Err(error) = std::fs::create_dir(&root) {
18323            assert!(
18324                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18325                "required cgroup test cannot execute: {error}"
18326            );
18327            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18328            return;
18329        }
18330        let placement = subc_cgroup::prepare_at(&root)
18331            .unwrap()
18332            .expect("delegated cgroup");
18333        let supervisor = Supervisor::new_for_test(
18334            Arc::new(Registry::default()),
18335            RestartPolicy::new(3, Duration::ZERO),
18336        )
18337        .with_cgroup_placement(Some(placement));
18338        let ready = dir.join("ready");
18339        let module = supervisor
18340            .spawn(ModuleSpec {
18341                module_id: "memory-tree".into(),
18342                program: PathBuf::from("/bin/sh"),
18343                args: vec![
18344                    "-c".into(),
18345                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18346                    "fixture".into(),
18347                    ready.display().to_string(),
18348                ],
18349                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18350                    .into_iter()
18351                    .map(|key| (key.into(), dir.display().to_string()))
18352                    .collect(),
18353                reserved: false,
18354                reserved_prefixes: Vec::new(),
18355                protocol: ModuleProtocol::None,
18356                overlap: Default::default(),
18357                carry_containment: false,
18358            })
18359            .unwrap();
18360        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18361        while !ready.exists() {
18362            assert!(
18363                tokio::time::Instant::now() < deadline,
18364                "children never started"
18365            );
18366            tokio::time::sleep(Duration::from_millis(10)).await;
18367        }
18368        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18369        let reading = &readings[0];
18370        assert!(
18371            reading.process_count.is_some_and(|count| count >= 4),
18372            "{reading:?}"
18373        );
18374        assert!(
18375            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18376            "{reading:?}"
18377        );
18378        module.stop().await.unwrap();
18379        drop(module);
18380        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18381        std::fs::remove_dir(&root).unwrap();
18382        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18383    }
18384
18385    fn running(pid: u32) -> bool {
18386        // An orphan can remain a zombie until the container init reaps it.
18387        std::fs::read_to_string(format!("/proc/{pid}/stat"))
18388            .ok()
18389            .and_then(|stat| {
18390                stat.rsplit_once(") ")
18391                    .map(|(_, rest)| rest.starts_with('Z'))
18392            })
18393            .is_some_and(|zombie| !zombie)
18394    }
18395
18396    #[tokio::test]
18397    async fn linux_teardown_reaps_the_grandchild() {
18398        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18399    }
18400
18401    #[tokio::test]
18402    async fn linux_shutdown_straggler_reaps_the_grandchild() {
18403        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18404    }
18405
18406    async fn teardown_tree(test_name: &str, shutdown: bool) {
18407        let dir = ScratchDir::new(test_name);
18408        let root = PathBuf::from(format!(
18409            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18410            std::process::id(),
18411            unix_ms_now()
18412        ));
18413        if let Err(error) = std::fs::create_dir(&root) {
18414            assert!(
18415                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18416                "required cgroup test cannot execute: {error}"
18417            );
18418            eprintln!(
18419                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18420                root.display()
18421            );
18422            return;
18423        }
18424        let placement = subc_cgroup::prepare_at(&root)
18425            .expect("prepare isolated kernel cgroup")
18426            .expect("isolated cgroup is delegated");
18427        let module_id = "tree-teardown";
18428        let module = placement
18429            .module_path(module_id)
18430            .expect("create isolated module cgroup");
18431        if !module.join("cgroup.kill").exists() {
18432            std::fs::remove_dir(&module).unwrap();
18433            std::fs::remove_dir(root.join("subc-modules")).unwrap();
18434            std::fs::remove_dir(&root).unwrap();
18435            assert!(
18436                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18437                "required cgroup.kill interface unavailable"
18438            );
18439            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18440            return;
18441        }
18442        let supervisor = Supervisor::new_for_test(
18443            Arc::new(Registry::default()),
18444            RestartPolicy::new(3, Duration::ZERO),
18445        )
18446        .with_cgroup_placement(Some(placement));
18447        let mut runtime = supervisor.runtime_config();
18448        runtime.child_roster = runtime
18449            .child_roster
18450            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18451        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18452        let pid_file = dir.join("grandchild.pid");
18453        let spec = ModuleSpec {
18454            module_id: module_id.to_string(),
18455            program: PathBuf::from("/bin/sh"),
18456            args: vec![
18457                "-c".into(),
18458                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18459                "fixture".into(),
18460                pid_file.display().to_string(),
18461            ],
18462            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18463                .into_iter()
18464                .map(|key| (key.to_string(), dir.display().to_string()))
18465                .collect(),
18466            reserved: false,
18467            reserved_prefixes: Vec::new(),
18468            protocol: ModuleProtocol::None,
18469            overlap: Default::default(),
18470            carry_containment: false,
18471        };
18472        let child =
18473            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18474        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18475        let grandchild: u32 = loop {
18476            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18477                if let Ok(pid) = contents.trim().parse() {
18478                    break pid;
18479                }
18480            }
18481            assert!(
18482                tokio::time::Instant::now() < deadline,
18483                "grandchild pid was not recorded"
18484            );
18485            tokio::time::sleep(Duration::from_millis(10)).await;
18486        };
18487        assert!(
18488            running(grandchild),
18489            "grandchild must be alive before teardown"
18490        );
18491        if shutdown {
18492            let mut child = child;
18493            crate::child_roster::end_children_for_daemon_shutdown(
18494                &runtime.child_roster,
18495                false,
18496                std::future::pending(),
18497            )
18498            .await;
18499            child.wait().await.expect("reap shutdown straggler");
18500        } else {
18501            drain_child_to_state(
18502                module_id,
18503                ModuleProtocol::None,
18504                StopNotice::NotSent,
18505                &Registry::default(),
18506                None,
18507                &snapshot,
18508                &runtime.terminal_ring,
18509                &SpawnEventFeed::default(),
18510                child,
18511                Duration::from_millis(100),
18512                ModuleState::Stopped,
18513                Some(false),
18514                &FailureContext::operator(),
18515            )
18516            .await
18517            .expect("real supervisor teardown");
18518        }
18519        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18520        while running(grandchild) && tokio::time::Instant::now() < deadline {
18521            tokio::time::sleep(Duration::from_millis(10)).await;
18522        }
18523        let survived = running(grandchild);
18524        // Kill a surviving grandchild so a failed test does not leave it behind.
18525        if survived {
18526            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18527            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18528            tokio::time::sleep(Duration::from_millis(100)).await;
18529        }
18530        if module.exists() {
18531            std::fs::remove_dir(&module).expect("remove empty module cgroup");
18532        }
18533        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18534        std::fs::remove_dir(&root).unwrap();
18535        assert!(
18536            !survived,
18537            "grandchild {grandchild} outlived module teardown"
18538        );
18539        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18540    }
18541}
18542
18543/// A slow live-children record write on the restart path must not stop the
18544/// daemon from answering the replacement module's authentication handshake.
18545#[cfg(all(test, unix))]
18546mod record_write_stall_tests {
18547    use super::*;
18548    use std::sync::atomic::{AtomicBool, Ordering};
18549    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18550
18551    /// `subc-client-rs` gives the whole handshake this long before the module
18552    /// exits with `Timeout { stage: ServerProof }`.
18553    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18554    /// How long the replacement's admission write is held: longer than the
18555    /// module's deadline, standing in for the delay a busy disk can add
18556    /// across the two syncs in that write.
18557    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18558
18559    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
18560    /// On macOS every module is launched through the privacy trampoline, and
18561    /// a module that is not a `#!` script is admitted only if its running
18562    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
18563    /// the shell soon replaces itself with `/bin/sleep`, which does not match
18564    /// the configured `/bin/sh`. The image is read only after the spawn's
18565    /// live-children record write, and here that write lasted long enough for
18566    /// the shell to have exec'd every time, so each process was refused and
18567    /// killed right after spawn, and the supervisor crash-looped it with
18568    /// backoff until its restart budget ran out. The restart this test
18569    /// measures would then never produce a module at all.
18570    fn sleeping_spec() -> ModuleSpec {
18571        ModuleSpec {
18572            module_id: "record-write-stall".into(),
18573            program: "/bin/sleep".into(),
18574            args: vec!["600".into()],
18575            env: vec![],
18576            reserved: false,
18577            reserved_prefixes: vec![],
18578            protocol: ModuleProtocol::None,
18579            overlap: Default::default(),
18580            carry_containment: false,
18581        }
18582    }
18583
18584    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18585    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18586        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18587        let supervisor =
18588            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18589                .with_live_children_record(home.join("live-children.json"));
18590        let module = supervisor.spawn(sleeping_spec()).unwrap();
18591        // `reported_pid` names a process only once it is confirmed as the
18592        // module (on macOS, once the trampoline's image check has passed), so
18593        // a process the daemon is about to refuse and kill never counts as
18594        // the running module, here or in the final wait below.
18595        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18596            loop {
18597                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18598                    .unwrap()
18599                    .reported_pid()
18600                {
18601                    return pid;
18602                }
18603                tokio::time::sleep(Duration::from_millis(10)).await;
18604            }
18605        })
18606        .await
18607        .expect("the first process was admitted as the module");
18608
18609        // The daemon's listener, served by this runtime's workers like the
18610        // supervisor's own tasks.
18611        let key = vec![0x42; 32];
18612        let daemon_id = [0x24; 16];
18613        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18614        let port = listener.local_addr().unwrap().port();
18615        let server = tokio::spawn(crate::server::serve_listener(
18616            listener,
18617            Arc::new(crate::router::Router::with_default_self_handler()),
18618            crate::server::ServerAuth::with_limits(
18619                key.clone(),
18620                daemon_id,
18621                "test",
18622                crate::server::DEFAULT_AUTH_DEADLINE,
18623                4,
18624            ),
18625        ));
18626
18627        // Hold the write that admits the replacement, once, and say when it
18628        // has started.
18629        let (held_tx, held_rx) = std::sync::mpsc::channel();
18630        let armed = AtomicBool::new(true);
18631        supervisor
18632            .child_roster
18633            .set_record_write_hook_for_test(move |live| {
18634                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18635                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18636                    let _ = held_tx.send(());
18637                    std::thread::sleep(HELD_RECORD_WRITE);
18638                }
18639            });
18640
18641        // The replacement module, as a separate process would be: its own
18642        // thread and runtime, so only the daemon's runtime can be stalled.
18643        let module_process = std::thread::spawn(move || {
18644            held_rx
18645                .recv_timeout(Duration::from_secs(30))
18646                .expect("the restart reached the replacement's record write");
18647            let conn = ConnectionInfo {
18648                schema: SCHEMA_VERSION,
18649                wire_version: None,
18650                endpoints: vec![Endpoint {
18651                    host: "127.0.0.1".to_owned(),
18652                    port,
18653                }],
18654                key,
18655                daemon_id,
18656                pid: std::process::id(),
18657                daemon_ver: "test".to_owned(),
18658            };
18659            tokio::runtime::Builder::new_current_thread()
18660                .enable_all()
18661                .build()
18662                .unwrap()
18663                .block_on(async {
18664                    let started = Instant::now();
18665                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18666                        .await
18667                        .unwrap();
18668                    let result =
18669                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18670                    (result, started.elapsed())
18671                })
18672        });
18673
18674        module.restart(Some(0)).await.unwrap();
18675        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18676            .await
18677            .unwrap();
18678        assert!(
18679            auth.is_ok(),
18680            "the daemon must answer a module's handshake while a restart waits on the \
18681             live-children record write; got {auth:?} after {elapsed:?}"
18682        );
18683
18684        tokio::time::timeout(Duration::from_secs(20), async {
18685            while lock_snapshot(&module.inner.snapshot)
18686                .unwrap()
18687                .reported_pid()
18688                .is_none_or(|pid| pid == old_pid)
18689            {
18690                tokio::time::sleep(Duration::from_millis(10)).await;
18691            }
18692        })
18693        .await
18694        .expect("the restart published its replacement");
18695        module.stop().await.unwrap();
18696        server.abort();
18697    }
18698}
18699
18700#[cfg(all(test, unix))]
18701mod retained_status_tests {
18702    use super::*;
18703
18704    #[tokio::test]
18705    async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18706        use crate::live_children::{ContainerIdentity, RetainedReason};
18707        for (reason, carried, expected) in [
18708            (
18709                RetainedReason::ContainerSurvivors(vec![4242]),
18710                true,
18711                Some("waiting: container survivors (pids [4242])".to_owned()),
18712            ),
18713            (
18714                RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18715                true,
18716                Some(
18717                    "waiting: container survivors could not be verified: member inspection denied"
18718                        .to_owned(),
18719                ),
18720            ),
18721            (RetainedReason::Survived, false, None),
18722        ] {
18723            let supervisor =
18724                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18725            let spec = ModuleSpec {
18726                module_id: "retained-status-fixture".into(),
18727                program: PathBuf::from("/not-spawned"),
18728                args: Vec::new(),
18729                env: Vec::new(),
18730                reserved: false,
18731                reserved_prefixes: Vec::new(),
18732                protocol: ModuleProtocol::None,
18733                overlap: ModuleOverlap::Exclusive,
18734                carry_containment: carried,
18735            };
18736            let module = supervisor
18737                .supervise_configured(spec.clone(), false)
18738                .unwrap();
18739            // No process is launched. Invalid group zero is deliberate: if a
18740            // status read wrongly rechecks, verification changes the reason
18741            // and fails the assertions without signalling any real process.
18742            let guard = supervisor
18743                .child_roster
18744                .admit(
18745                    spec.module_id,
18746                    0,
18747                    ModuleProtocol::None,
18748                    None,
18749                    crate::child_roster::RecordedIdentity {
18750                        container: carried.then_some(ContainerIdentity {
18751                            pgid: Some(0),
18752                            created_start_time: Some(0),
18753                            cgroup_path: None,
18754                            ..ContainerIdentity::test_metadata()
18755                        }),
18756                        ..crate::child_roster::RecordedIdentity::default()
18757                    },
18758                )
18759                .unwrap();
18760            guard.retain(reason);
18761            let status = module.status().unwrap();
18762            assert_eq!(status.waiting_for_previous_detail, expected.clone());
18763            assert_eq!(status.health.detail, expected);
18764            assert!(!status.process_alive);
18765        }
18766    }
18767}