Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68/// Command-line flag used by supervised modules to find subc.
69///
70/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
71/// The path points at the TCP+key connection file; it is not an ambient signal and
72/// is never inherited by standalone children.
73pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
79/// to contain a real crash loop (which respawns in seconds) and short enough
80/// that unrelated crashes hours apart never accumulate into a permanent stop.
81const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82/// How long a drain waits for already-dispatched requests to finalize before
83/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
84/// not RPC-scale: the original 2s value silently cut nearly every real tool
85/// call at the fence, making the wait-for-finalize design decorative for the
86/// workloads it existed for. Quiescence short-circuits, so an idle module
87/// restarts immediately regardless of this value; the budget is spent only
88/// when a genuine in-flight request is worth finishing. Per-module override:
89/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
90/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
91/// where a stuck request will never settle).
92pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96/// How long a drain waits after one busy-gauge probe completes before it asks
97/// again.
98///
99/// A drain re-probes on this short interval of its own rather than on the
100/// module's health cadence. The default cadence (30 s) equals the default
101/// drain ceiling, so a cadence-paced drain that found the module busy at its
102/// first probe never asked a second time: it waited out the whole ceiling and
103/// forced GOODBYE even after the gauges had read zero. The interval is counted
104/// from the moment the previous probe finished, and probes run one at a time,
105/// so a slow module is never asked more often than it can answer. Periodic
106/// health checks outside a drain keep using the configured cadence.
107const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108/// How long a restart waits for the exited process's output readers.
109///
110/// The restart does not depend on either output reader finishing. A stderr
111/// reader still running at this bound is left running, and its later output goes
112/// into the exited process's own section of the stderr ring (see
113/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
114/// bound no longer decides whether a crash's last lines are kept: under load
115/// the reader may simply not have been scheduled yet, and cutting it there
116/// discarded exactly the lines that explained the crash.
117///
118/// What the bound still decides is when the tail starts reporting
119/// `Incomplete`: a pipe open past it usually means a descendant of the exited
120/// process still holds it, and the tail cannot claim to be whole until that
121/// pipe closes. That is also the only case in which the wait costs the restart
122/// anything, because a pipe with no other holder reaches EOF when the process
123/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
124/// `Captured` at EOF with nothing lost.
125///
126/// The stdout reader carries no ring, only the capture file. It also continues
127/// to EOF after this bound: child exit closes the writer but does not mean the
128/// reader has consumed all buffered bytes. Late output from either pipe can
129/// therefore reach the shared capture file after the successor starts.
130const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
132pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
135/// dropped because the subscriber stopped draining its frames. The detail's
136/// `first_undelivered_cursor` names the first event it did not receive; the
137/// client resubscribes from the last cursor it did receive.
138pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143    deadline: tokio::time::Instant,
144    expected: Option<subc_os::FileIdentity>,
145    trampoline: Option<subc_os::FileIdentity>,
146    script: bool,
147    module_id: String,
148    #[cfg(test)]
149    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
155    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
156    let path = std::env::current_exe()
157        .unwrap()
158        .parent()
159        .unwrap()
160        .parent()
161        .unwrap()
162        .join("privacy-trampoline-fixture");
163    // Without the fixture every macOS spawn is refused, and the tests that
164    // spawn fail later as a module in state Failed, which names the wrong
165    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
166    assert!(
167        path.exists(),
168        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169         --bins --features test-support` or `cargo test -p subc-daemon` first",
170        path.display()
171    );
172    path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177    use std::io::Read;
178    let mut probe = std::process::Command::new(path)
179        .args(["__disclaim-exec", "--probe"])
180        .stdin(Stdio::null())
181        .stdout(Stdio::piped())
182        .stderr(Stdio::piped())
183        .spawn()
184        .map_err(|error| {
185            format!(
186                "privacy trampoline probe failed for {}: {error}",
187                path.display()
188            )
189        })?;
190    let deadline = std::time::Instant::now() + Duration::from_secs(5);
191    let status = loop {
192        match probe.try_wait() {
193            Ok(Some(status)) => break status,
194            Ok(None) if std::time::Instant::now() < deadline => {
195                std::thread::sleep(Duration::from_millis(5))
196            }
197            result => {
198                let _ = probe.kill();
199                let _ = probe.wait();
200                return Err(format!(
201                    "privacy trampoline probe failed or timed out for {}: {result:?}",
202                    path.display()
203                ));
204            }
205        }
206    };
207    let mut answer = String::new();
208    if let Some(stdout) = probe.stdout.take() {
209        let _ = stdout.take(256).read_to_string(&mut answer);
210    }
211    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212        return Ok(());
213    }
214    let mut diagnostic = String::new();
215    if let Some(stderr) = probe.stderr.take() {
216        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217    }
218    let cause = diagnostic
219        .trim()
220        .strip_prefix("ck-subc: own privacy identity refused: ")
221        .unwrap_or("binary does not implement the privacy trampoline protocol");
222    Err(format!(
223        "{cause}: probe of {} exited {status}",
224        path.display()
225    ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230    spec: &ModuleSpec,
231    roster: &ChildRoster,
232) -> Result<
233    (
234        Command,
235        Option<PrivacyExec>,
236        subc_os::privacy_identity::ExecAcknowledgement,
237    ),
238    SuperviseError,
239> {
240    let failure = |cause: String| {
241        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242        SuperviseError::Spawn {
243            program: spec.program.clone(),
244            source: io::Error::other(cause),
245            cgroup_path: None,
246        }
247    };
248    let trampoline = roster.privacy_trampoline().map_err(failure)?;
249    // Resolve PATH with the same environment the Command will receive. For
250    // scripts retain the existing orphan-identity rule: the kernel chooses
251    // the interpreter, and its observed image is the one recorded. Do not
252    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
253    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254        let path = spec
255            .env
256            .iter()
257            .find(|(key, _)| key == "PATH")
258            .map(|(_, value)| std::ffi::OsString::from(value))
259            .or_else(|| std::env::var_os("PATH"))
260            .unwrap_or_else(|| "/usr/bin:/bin".into());
261        std::env::split_paths(&path)
262            .map(|dir| dir.join(&spec.program))
263            .find(|path| path.is_file())
264            .unwrap_or_else(|| spec.program.clone())
265    } else {
266        spec.program.clone()
267    };
268    let expected = subc_os::file_identity(&program);
269    let trampoline_image = subc_os::file_identity(&trampoline);
270    let script = {
271        use std::io::Read;
272        let mut prefix = [0u8; 2];
273        std::fs::File::open(&program)
274            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275    };
276    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277        return Err(failure(
278            "privacy identity module executable is missing or is the trampoline itself".to_string(),
279        ));
280    }
281    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282        .map_err(|error| failure(error.to_string()))?;
283    let reader =
284        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285    let mut command = Command::new(&trampoline);
286    command
287        .arg("__disclaim-exec")
288        .arg(ack.fd().to_string())
289        .arg(&program);
290    ack.install(command.as_std_mut());
291    Ok((
292        command,
293        Some(PrivacyExec {
294            reader,
295            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296            expected,
297            trampoline: trampoline_image,
298            script,
299            module_id: spec.module_id.clone(),
300            #[cfg(test)]
301            confirmation_pause: spec
302                .env
303                .iter()
304                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305                .map(|(_, address)| (address.clone(), None)),
306        }),
307        ack,
308    ))
309}
310
311struct SupervisedChild {
312    child: Child,
313    exec: crate::activity::ExecOwner,
314    #[cfg(windows)]
315    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
316    #[cfg(target_os = "macos")]
317    privacy_exec: Option<PrivacyExec>,
318    /// Set once this launch's exec acknowledgement confirms the module image.
319    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
320    /// `subc_os::privacy_identity`), which then replaces itself with the
321    /// module. The supervisor owns and can kill that pid from spawn, but
322    /// status readers report it only after this latch is set, so nothing
323    /// reports the trampoline's image as the module's.
324    #[cfg(target_os = "macos")]
325    report_ready: Arc<OnceLock<()>>,
326    /// Supervisor failure detail retained for the terminal record, including
327    /// image refusals and first-HELLO timeouts after a successful spawn.
328    spawn_failure: Option<String>,
329    /// The protocol this process was launched with. A reload can store a new
330    /// launch spec with a different protocol, but that takes effect only at the
331    /// next spawn, so this process keeps being handled by the protocol it
332    /// actually speaks.
333    protocol: ModuleProtocol,
334    /// This process's cgroup name: a bounded module/slot label followed by a
335    /// spawn suffix unique to this process (when cgroup placement is on). A
336    /// retired process in a slot may still be draining when a later one is
337    /// spawned into that slot, so the suffix keeps the later process out of
338    /// the retired one's cgroup, which is the domain a kill applies to.
339    #[cfg(target_os = "linux")]
340    module_id: String,
341    #[cfg(target_os = "linux")]
342    cgroup_placement: Option<subc_cgroup::Placement>,
343    /// The job that contains this child and every process it spawns (issue #109).
344    ///
345    /// Dropping this handle is what reaps a surviving tree when no supervisor
346    /// code runs — a daemon crash — because the job carries
347    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
348    ///
349    /// That limit is not crash-only, and the difference is worth knowing: a
350    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
351    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
352    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
353    /// module at once. Before this change they survived that, saw EOF on the
354    /// control socket, and ran their own teardown; Unix keeps that path
355    /// deliberately, so a module can seal a WAL or close a capture rather than
356    /// be killed mid-write. So this trades graceful teardown on every Windows
357    /// daemon stop for containment on a crash, which is the right way round
358    /// today: orphaned GPU workers are a reported, recurring problem, and the
359    /// modules that write most heavily do not run on Windows.
360    ///
361    /// The fix is a real Windows stop path — the daemon draining before it
362    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
363    /// reaches only what the drain left behind, which is what it should reach.
364    #[cfg(windows)]
365    job: Option<subc_jobobject::JobObject>,
366    stdout_pump: Option<JoinHandle<()>>,
367    stderr_pump: Option<StderrPump>,
368    stderr_ring: Arc<Mutex<StderrRing>>,
369    /// Conservative monotonic bound sampled before `command.spawn()`, not when
370    /// identity recording finishes or the spawn generation is published.
371    spawned_at: Instant,
372    /// Ordinary spawns must send their first HELLO within this budget. Once
373    /// accepted, a later disconnect must not restart the registration clock.
374    registration_deadline: Option<Instant>,
375    registration_watch: Option<RegistrationWatch>,
376    spawned_at_ms: u64,
377    spawned_from: PathBuf,
378    launch_spec_digest: LaunchSpecDigest,
379    spawned_file_identity: Option<SpawnedFileIdentity>,
380    process_start_time: Option<u64>,
381    process_identity: Option<ProcessIdentity>,
382    pid: u32,
383    /// This process's entry in the daemon's child roster, released when the
384    /// process is reaped or this handle is dropped.
385    roster_guard: Option<crate::child_roster::RosterGuard>,
386}
387
388impl SupervisedChild {
389    fn registered_since_spawn(&self) -> bool {
390        self.registration_watch
391            .as_ref()
392            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
393    }
394
395    fn id(&self) -> Option<u32> {
396        Some(self.pid)
397    }
398
399    fn process_identity(&self) -> Option<ProcessIdentity> {
400        self.process_identity
401    }
402
403    async fn wait(&mut self) -> io::Result<ExitStatus> {
404        #[cfg(target_os = "macos")]
405        self.confirm_privacy_exec().await;
406        // The roster entry is NOT released here. A daemon shutdown waits for the
407        // roster to empty and then exits the process, so releasing at the reap
408        // let it exit before the exit handler wrote this child's terminal record
409        // (the stderr drain and snapshot update sit in between), and the
410        // shutdown's own `daemon_shutdown` record was intermittently lost. The
411        // caller releases it after recording the exit (`release_roster`), and
412        // dropping the handle releases it too.
413        let result = self.child.wait().await;
414        #[cfg(target_os = "linux")]
415        if result.is_ok() {
416            if let Some(placement) = self.cgroup_placement.as_ref() {
417                cleanup_reaped_cgroup(placement, &self.module_id).await;
418                // Keep ownership while awaiting kernel population changes: a
419                // drain timeout may cancel this wait and then escalate/reap.
420                self.cgroup_placement = None;
421            }
422        }
423        result
424    }
425
426    #[cfg(target_os = "macos")]
427    async fn confirm_privacy_exec(&mut self) {
428        let Some(pending) = &mut self.privacy_exec else {
429            return;
430        };
431        let result = tokio::time::timeout_at(pending.deadline, async {
432            let mut record = Vec::new();
433            loop {
434                let mut ready = pending.reader.readable().await?;
435                let read = ready.try_io(|reader| {
436                    use std::io::Read;
437                    let mut reader = reader.get_ref();
438                    let mut buffer = [0u8; 256];
439                    reader.read(&mut buffer).map(|count| (count, buffer))
440                });
441                match read {
442                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
443                    Ok(Ok((count, buffer))) => {
444                        if record.len() + count > 1024 {
445                            return Err(io::Error::other(
446                                "privacy exec refusal record is too long",
447                            ));
448                        }
449                        record.extend_from_slice(&buffer[..count]);
450                    }
451                    Ok(Err(error)) => return Err(error),
452                    Err(_) => continue,
453                }
454            }
455        })
456        .await;
457        #[cfg(test)]
458        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
459            use tokio::io::{AsyncReadExt, AsyncWriteExt};
460            if stream.is_none() {
461                *stream = Some(
462                    tokio::net::TcpStream::connect(address.as_str())
463                        .await
464                        .unwrap(),
465                );
466                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
467                stream
468                    .as_mut()
469                    .unwrap()
470                    .write_all(&self.pid.to_ne_bytes())
471                    .await
472                    .unwrap();
473            }
474            let mut reply = [0];
475            stream
476                .as_mut()
477                .unwrap()
478                .read_exact(&mut reply)
479                .await
480                .unwrap();
481            pending.confirmation_pause = None;
482            if &reply == b"F" {
483                Ok(Err(io::Error::other("injected late exec refusal")))
484            } else {
485                assert_eq!(&reply, b"X");
486                result
487            }
488        } else {
489            result
490        };
491        // Keep the reader in self across await: select cancellation must not
492        // discard the handshake or reset its original five-second deadline.
493        let pending = self.privacy_exec.as_ref().expect("pending exec");
494        let cause = match result {
495            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
496            Ok(Err(error)) => Some(format!(
497                "privacy identity exec acknowledgement failed: {error}"
498            )),
499            Ok(Ok(record)) if !record.is_empty() => Some(
500                std::str::from_utf8(&record)
501                    .ok()
502                    .and_then(|record| {
503                        record
504                            .trim()
505                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
506                    })
507                    .filter(|cause| !cause.is_empty())
508                    .unwrap_or("invalid privacy exec refusal record")
509                    .to_string(),
510            ),
511            Ok(Ok(_)) => match self.child.try_wait() {
512                // Empty EOF is the exec acknowledgement. A real module may exit
513                // immediately, including with a reserved trampoline status; no
514                // image is admitted, and its ordinary exit contract stays intact.
515                Ok(Some(_status)) => None,
516                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
517                Ok(None) => {
518                    let image = observe_spawned_image(self.pid);
519                    if let Some(image) = image.filter(|image| {
520                        image.executable.is_some()
521                            && image.executable != pending.trampoline
522                            && (image.executable == pending.expected || pending.script)
523                    }) {
524                        if let Err(error) = self
525                            .roster_guard
526                            .as_ref()
527                            .map(|guard| guard.confirm_executable(image))
528                            .transpose()
529                        {
530                            Some(format!(
531                                "could not durably record confirmed module image: {error}"
532                            ))
533                        } else {
534                            let _ = self.report_ready.set(());
535                            info!(module_id = %pending.module_id, pid = self.pid,
536                                "module spawned with own privacy identity (responsibility disclaimed)");
537                            None
538                        }
539                    } else if image.is_none()
540                        || image.is_some_and(|image| image.executable.is_none())
541                    {
542                        // A process can exit between try_wait and the kernel
543                        // image read. Empty EOF already acknowledged exec, so
544                        // preserve that module's ordinary exit rather than
545                        // mislabel a disappearing image as trampoline refusal.
546                        // Keep pending in self across await so cancellation does
547                        // not discard validation or reset its original deadline.
548                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
549                            Ok(Ok(_status)) => None,
550                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
551                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
552                        }
553                    } else {
554                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
555                    }
556                }
557            },
558        };
559        let pending = self.privacy_exec.take().expect("pending exec");
560        // Exec confirmation describes launch success, not roster retirement.
561        // Keep even a reaped child recorded until its terminal history exists.
562        self.exec.0.finish(if cause.is_some() {
563            crate::activity::ExecState::Refused
564        } else {
565            crate::activity::ExecState::CommittedExec
566        });
567        if let Some(cause) = cause {
568            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
569            self.spawn_failure = Some(cause);
570            // No image is admitted on failure. Reach the entire fresh process
571            // group, including a module which spawned a helper before refusal.
572            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
573                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
574            }
575            let _ = self.child.start_kill();
576        }
577    }
578
579    /// Releases this child's daemon-shutdown roster entry once its exit has
580    /// been recorded. The pid is already reaped and free for reuse, so the
581    /// entry must not outlive the record any longer than that.
582    fn release_roster(&mut self) {
583        self.roster_guard = None;
584    }
585
586    /// Kill the child and, where containment is available, its process tree.
587    ///
588    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
589    /// helper process leaked the helper — the Synapse embedding module's CUDA
590    /// worker holds the GPU allocation, so the leak cost VRAM until the next
591    /// restart of something else. Terminating the job reaches grandchildren that
592    /// a tree walk cannot, including one whose parent has already exited and
593    /// been reparented away.
594    ///
595    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
596    /// direct-child kill still decides the outcome, so containment can never
597    /// change whether a module is reported as stopped.
598    fn start_kill(&mut self) -> io::Result<()> {
599        #[cfg(windows)]
600        if let Some(job) = &self.job {
601            if let Err(error) = job.terminate() {
602                debug!(
603                    error = %error,
604                    "job termination failed; the direct-child kill still owns the outcome"
605                );
606            }
607        }
608        #[cfg(target_os = "linux")]
609        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
610        self.child.start_kill()
611    }
612
613    async fn drain_stderr(&mut self, module_id: &str) {
614        if let Some(pump) = self.stdout_pump.take() {
615            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
616        }
617
618        let Some(pump) = self.stderr_pump.take() else {
619            return;
620        };
621        settle_stderr_pump(
622            module_id,
623            &self.stderr_ring,
624            pump,
625            STDERR_PUMP_DRAIN_TIMEOUT,
626        )
627        .await;
628    }
629}
630
631/// Bound the restart's wait, not the lifetime of the exited child's stdout
632/// reader. Slow pipe reads or capture writes must not turn the bound into lost
633/// output; dropping the handle lets the reader finish at EOF, like stderr.
634async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
635    match timeout(bound, &mut pump).await {
636        Ok(Ok(())) => {}
637        Ok(Err(error)) => {
638            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
639        }
640        Err(_) => {
641            drop(pump);
642            warn!(
643                module_id,
644                waited = ?bound,
645                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
646            );
647        }
648    }
649}
650
651/// The reader task for one process's stderr, with the ring generation its
652/// lines are attributed to.
653struct StderrPump {
654    task: JoinHandle<()>,
655    generation: u64,
656}
657
658/// Retire an exited process's stderr reader and wait up to `bound` for it to
659/// reach EOF. A reader still running at the bound is detached, not stopped: it
660/// keeps filling the exited process's section of the ring until its pipe
661/// closes, and the tail reads `Incomplete` until then. See
662/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
663async fn settle_stderr_pump(
664    module_id: &str,
665    ring: &Arc<Mutex<StderrRing>>,
666    pump: StderrPump,
667    bound: Duration,
668) {
669    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
670    let StderrPump {
671        mut task,
672        generation,
673    } = pump;
674    lock().retire_pump(generation);
675    match timeout(bound, &mut task).await {
676        Ok(Ok(())) => {}
677        Ok(Err(err)) => {
678            let mut ring = lock();
679            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
680            ring.finish_pump(generation);
681            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
682        }
683        Err(_) => {
684            // Dropping the handle detaches the task; it ends at EOF on its pipe.
685            drop(task);
686            lock().mark_pump_late(
687                generation,
688                format!(
689                    "stderr of the exited process had not reached EOF {bound:?} after it was \
690                     retired (a descendant may still hold the pipe open); lines it still \
691                     writes are kept in that process's section"
692                ),
693            );
694            warn!(
695                module_id,
696                waited = ?bound,
697                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
698            );
699        }
700    }
701}
702
703fn registration_release_events() -> &'static watch::Sender<u64> {
704    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
705    EVENTS.get_or_init(|| {
706        let (sender, _receiver) = watch::channel(0);
707        sender
708    })
709}
710
711pub(crate) fn notify_registration_release() {
712    let events = registration_release_events();
713    let next_generation = (*events.borrow()).wrapping_add(1);
714    events.send_replace(next_generation);
715}
716
717/// How to launch one singleton module process.
718#[derive(Debug, Clone, PartialEq, Eq)]
719pub struct ModuleSpec {
720    pub module_id: String,
721    pub program: PathBuf,
722    pub args: Vec<String>,
723    pub env: Vec<(String, String)>,
724    /// When true this is a reserved module: each spawn gets a fresh one-time launch
725    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
726    /// process can register this module_id (a security-boundary module like the
727    /// credential vault must not be impersonable while it is down/restarting).
728    pub reserved: bool,
729    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
730    /// Prefixes come from daemon config and must end in `:` before they reach the
731    /// supervisor; the owner module's current spawn nonce authorizes claims under
732    /// each prefix.
733    pub reserved_prefixes: Vec<String>,
734    /// The wire protocol this module speaks, as DECLARED in daemon config.
735    ///
736    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
737    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
738    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
739    /// and NO launch nonce, and a clean exit the daemon did not request is
740    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
741    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
742    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
743    /// because a process ignores an environment variable it does not read.
744    ///
745    /// The argument is the part that cannot be "harmless to a process that
746    /// ignores it": a stock binary exits on an unknown flag before it listens
747    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
748    /// first conformance run against this mode found it. The nonce is withheld
749    /// because a process that will never present it gains nothing from holding
750    /// it, and a secret in the environment of a process that does not need it is
751    /// a leak surface for no benefit.
752    pub protocol: ModuleProtocol,
753    /// Whether two processes of this module may run at once, which is what a
754    /// blue/green swap does for the length of its overlap. Declared in daemon
755    /// config because the daemon must be able to answer it while the module is
756    /// down, and so a module cannot talk itself into it after registering.
757    pub overlap: ModuleOverlap,
758}
759
760/// Only the configured launch inputs enter this value; spawned connection paths,
761/// nonces and inherited environment never do. Its bytes remain daemon-private.
762#[derive(Clone, PartialEq, Eq)]
763pub(crate) struct LaunchSpecDigest([u8; 32]);
764
765impl fmt::Debug for LaunchSpecDigest {
766    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
767        f.write_str("LaunchSpecDigest([redacted])")
768    }
769}
770
771impl LaunchSpecDigest {
772    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
773        fn field(hash: &mut Sha256, bytes: &[u8]) {
774            hash.update((bytes.len() as u64).to_le_bytes());
775            hash.update(bytes);
776        }
777        let mut hash = Sha256::new();
778        if let Some(program) = spec.program.to_str() {
779            field(&mut hash, b"utf8-program");
780            field(&mut hash, program.as_bytes());
781        } else {
782            // The daemon config is UTF-8, but in-process callers may provide
783            // native paths: lossy conversion would collapse distinct programs.
784            #[cfg(unix)]
785            {
786                use std::os::unix::ffi::OsStrExt;
787                field(&mut hash, b"unix-program");
788                field(&mut hash, spec.program.as_os_str().as_bytes());
789            }
790            #[cfg(windows)]
791            {
792                use std::os::windows::ffi::OsStrExt;
793                field(&mut hash, b"windows-program");
794                let bytes = spec
795                    .program
796                    .as_os_str()
797                    .encode_wide()
798                    .flat_map(u16::to_le_bytes)
799                    .collect::<Vec<_>>();
800                field(&mut hash, &bytes);
801            }
802            #[cfg(not(any(unix, windows)))]
803            field(&mut hash, spec.program.to_string_lossy().as_bytes());
804        }
805        hash.update((spec.args.len() as u64).to_le_bytes());
806        for arg in &spec.args {
807            field(&mut hash, arg.as_bytes());
808        }
809        let mut env = spec.env.iter().collect::<Vec<_>>();
810        env.sort_unstable();
811        hash.update((env.len() as u64).to_le_bytes());
812        for (key, value) in env {
813            field(&mut hash, key.as_bytes());
814            field(&mut hash, value.as_bytes());
815        }
816        field(
817            &mut hash,
818            match spec.protocol {
819                ModuleProtocol::Subc => b"subc",
820                ModuleProtocol::None => b"none",
821            },
822        );
823        Self(hash.finalize().into())
824    }
825}
826
827/// Whether a module tolerates a second process of itself running alongside.
828///
829/// Most modules are single-writer on their store (a WAL, a capture log, a
830/// resident index behind a writer barrier), and two processes on one store
831/// corrupt it. So a swap, which overlaps the old and new process by design,
832/// is refused unless the module's config opts in with `overlap: "safe"`.
833#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
834pub enum ModuleOverlap {
835    /// Never run two processes of this module at once. The default.
836    #[default]
837    Exclusive,
838    /// The module has said a second process of itself is harmless for the
839    /// length of a swap.
840    ///
841    /// Declare it only if a second instance can run for a few seconds without
842    /// touching ANY single-writer store: every database, WAL, index, projector
843    /// and scheduled job the module owns. A lease on part of that state is not
844    /// enough. broca's session lease guards WAL appends while its run index, its
845    /// store projector and its archive fold timer (which unlinks live WAL files)
846    /// stay single-writer, so broca is exclusive despite holding a lease. The
847    /// refusal only fires after this has been decided, so the decision is the
848    /// check.
849    Safe,
850}
851
852impl ModuleOverlap {
853    pub fn as_str(self) -> &'static str {
854        match self {
855            Self::Exclusive => "exclusive",
856            Self::Safe => "safe",
857        }
858    }
859}
860
861/// Environment variable telling a spawned module which case it was started
862/// for, before it sends HELLO. Only a swap candidate carries it, as
863/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
864///
865/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
866/// longer because nobody waits on it, while a plain restart must flip ready
867/// quickly because callers see `module_warming` until it does. Absence means
868/// plain restart, the safe reading. The daemon trusts nothing about it; the
869/// candidate is proven by its launch nonce at HELLO.
870pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
871/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
872pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
873/// How long a swap waits for its candidate to register and declare itself
874/// ready when the operator does not say. A module warming as a swap candidate
875/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
876/// daemon allows that plus time to start the process and send HELLO.
877pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
878
879/// Bounded restart policy for crash exits.
880///
881/// `max_restarts` is the number of replacement processes allowed after the
882/// initial spawn WITHIN `window`. After that many crash restarts inside one
883/// window the module enters [`ModuleState::Failed`] and the supervisor stops
884/// the crash loop.
885///
886/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
887/// and that only survived because crashes were rare: a module that crashed
888/// three times across a week was disabled forever by crashes that had nothing
889/// to do with each other. That stopped being survivable once modules began
890/// exiting non-zero whenever the daemon's connection to them drops, because
891/// then every daemon-side connection drop spends a unit of the same budget and
892/// one flappy hour permanently stops a healthy module. Restarts older than
893/// `window` release their slot, so a module that crashed twice yesterday has a
894/// full budget today, while a genuine crash loop -- which is fast by
895/// definition -- still reaches the cap and stops.
896#[derive(Debug, Clone, Copy, PartialEq, Eq)]
897pub struct RestartPolicy {
898    pub max_restarts: u32,
899    /// Base delay before a crash replacement. The actual delay escalates with
900    /// the number of recent crash replacements and is capped by `max_backoff`.
901    pub backoff: Duration,
902    /// Maximum delay before a crash replacement.
903    pub max_backoff: Duration,
904    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
905    /// budget effectively infinite (nothing is ever in-window), which is why
906    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
907    pub window: Duration,
908}
909
910impl RestartPolicy {
911    /// A policy with the default crash window. Callers that care about the
912    /// window say so with [`Self::with_window`]; the ones that do not are
913    /// asking for the standard rate limit, not for no limit.
914    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
915        Self {
916            max_restarts,
917            backoff,
918            max_backoff: DEFAULT_MAX_BACKOFF,
919            window: DEFAULT_RESTART_WINDOW,
920        }
921    }
922
923    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
924        self.max_backoff = max_backoff;
925        self
926    }
927
928    pub fn with_window(mut self, window: Duration) -> Self {
929        self.window = window;
930        self
931    }
932
933    /// Calculate the capped exponential delay for the next crash replacement.
934    /// `restart_in_window` is zero for the first replacement after an operator
935    /// action (restart, reload, re-enable) cleared the crash ring, or after all
936    /// older crash replacements have aged out of the window.
937    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
938        if self.backoff.is_zero() || self.max_backoff.is_zero() {
939            return Duration::ZERO;
940        }
941
942        let mut delay = self.backoff;
943        for _ in 0..restart_in_window {
944            if delay >= self.max_backoff {
945                return self.max_backoff;
946            }
947            delay = delay
948                .checked_mul(10)
949                .unwrap_or(self.max_backoff)
950                .min(self.max_backoff);
951        }
952        delay.min(self.max_backoff)
953    }
954
955    /// The one sentence that explains a budget-exhausted stop, used for both the
956    /// log line and the terminal record so the two cannot drift. It names the
957    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
958    /// exactly what this budget is not.
959    fn budget_exhausted_detail(&self) -> String {
960        format!(
961            "crash budget exhausted: max_restarts={} within window_secs={}",
962            self.max_restarts,
963            self.window.as_secs()
964        )
965    }
966}
967
968impl Default for RestartPolicy {
969    fn default() -> Self {
970        Self {
971            max_restarts: DEFAULT_MAX_RESTARTS,
972            backoff: DEFAULT_BACKOFF,
973            max_backoff: DEFAULT_MAX_BACKOFF,
974            window: DEFAULT_RESTART_WINDOW,
975        }
976    }
977}
978
979#[derive(Debug, Clone, Copy, PartialEq, Eq)]
980struct CrashRestartSchedule {
981    restart_in_window: u32,
982    delay: Duration,
983}
984
985/// Whether the daemon itself will bring this module back after the exit being
986/// handled: it is enabled AND its in-window crash restarts are below the cap.
987///
988/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
989/// the window are dropped here rather than by a timer, so the count is right
990/// the moment somebody asks and no bookkeeping runs for idle modules.
991fn daemon_will_restart(
992    state: &mut SupervisorSnapshot,
993    policy: &RestartPolicy,
994    now: Instant,
995) -> bool {
996    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
997}
998
999const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1000const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1001const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1002const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1003
1004#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1005pub enum HealthAction {
1006    Report,
1007    Restart,
1008    Alert,
1009}
1010
1011impl fmt::Display for HealthAction {
1012    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1013        f.write_str(match self {
1014            Self::Report => "report",
1015            Self::Restart => "restart",
1016            Self::Alert => "alert",
1017        })
1018    }
1019}
1020
1021#[derive(Debug, Clone, PartialEq, Eq)]
1022pub struct HealthConfig {
1023    /// Optional loopback HTTP endpoint for a managed non-wire process.
1024    /// Changing it applies live on rescan; the process protocol changes only
1025    /// at its next spawn.
1026    pub http: Option<String>,
1027    pub cadence: Duration,
1028    pub deadline: Duration,
1029    pub failure_threshold: u32,
1030    pub on_degraded: HealthAction,
1031    pub on_failing: HealthAction,
1032    pub critical: bool,
1033}
1034
1035impl Default for HealthConfig {
1036    fn default() -> Self {
1037        Self {
1038            http: None,
1039            cadence: DEFAULT_HEALTH_CADENCE,
1040            deadline: DEFAULT_HEALTH_DEADLINE,
1041            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1042            on_degraded: HealthAction::Report,
1043            on_failing: HealthAction::Report,
1044            critical: false,
1045        }
1046    }
1047}
1048
1049/// The supervisor's view of one module's health, relayed to clients over
1050/// channel-0 and rendered by `ck health`.
1051///
1052/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1053/// stated here rather than only at the wire type a consumer reads. A reader can
1054/// look up what `None` means; only a writer can silently change it, and the
1055/// writer has no reason to go looking at a downstream contract before editing.
1056///
1057/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1058/// back to `None` on re-registration precisely so a respawned module does not
1059/// carry its predecessor's timestamp — so an old value and an absent one call for
1060/// opposite readings, and anything that defaulted this to a number would make a
1061/// never-probed module indistinguishable from one probed at the epoch.
1062///
1063/// `detail` and `metrics` are `None` when the module published none on this
1064/// probe, which does not mean it reported nothing wrong — it is also the shape
1065/// when the probe never reached it. `last_probe_ms` is what separates those.
1066#[derive(Debug, Clone, PartialEq)]
1067pub struct ModuleHealthStatus {
1068    pub status: SupervisorHealthStatus,
1069    pub last_probe_ms: Option<u64>,
1070    pub detail: Option<String>,
1071    pub metrics: Option<Value>,
1072    pub consecutive_failures: u32,
1073    /// Number of replies received after a recurring health probe's deadline.
1074    /// Unlike a timeout, every increment proves the module was alive.
1075    pub late_answer_count: u64,
1076    /// End-to-end latency of the newest late reply, measured from probe start.
1077    pub last_late_answer_latency_ms: Option<u64>,
1078    pub last_action: Option<String>,
1079    /// Set together with `last_action`; the pair moves as one, and both being
1080    /// absent means no escalation has ever been taken rather than that the last
1081    /// one succeeded.
1082    pub last_action_ms: Option<u64>,
1083}
1084
1085impl Default for ModuleHealthStatus {
1086    fn default() -> Self {
1087        Self {
1088            status: SupervisorHealthStatus::Unknown,
1089            last_probe_ms: None,
1090            detail: None,
1091            metrics: None,
1092            consecutive_failures: 0,
1093            late_answer_count: 0,
1094            last_late_answer_latency_ms: None,
1095            last_action: None,
1096            last_action_ms: None,
1097        }
1098    }
1099}
1100
1101/// Typed lifecycle state for a supervised module.
1102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1103pub enum ModuleState {
1104    Starting,
1105    Running,
1106    Unresponsive,
1107    Restarting,
1108    Draining,
1109    Stopped,
1110    Failed,
1111    Disabled,
1112}
1113
1114impl fmt::Display for ModuleState {
1115    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1116        f.write_str(match self {
1117            Self::Starting => "starting",
1118            Self::Running => "running",
1119            Self::Unresponsive => "unresponsive",
1120            Self::Restarting => "restarting",
1121            Self::Draining => "draining",
1122            Self::Stopped => "stopped",
1123            Self::Failed => "failed",
1124            Self::Disabled => "disabled",
1125        })
1126    }
1127}
1128
1129/// Supervisor classification of a child-process exit.
1130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1131pub enum ExitKind {
1132    Clean,
1133    Crash,
1134    DeliberateSeverance,
1135}
1136
1137impl From<ExitKind> for TerminalExitKind {
1138    fn from(kind: ExitKind) -> Self {
1139        match kind {
1140            ExitKind::Clean => Self::Clean,
1141            ExitKind::Crash => Self::Crash,
1142            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1143        }
1144    }
1145}
1146
1147/// Exact process identity retained when a supervised module registers its
1148/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub(crate) struct ProcessIdentity {
1151    pub(crate) pid: u32,
1152    pub(crate) start_time: u64,
1153}
1154
1155/// Last observed child exit, if any.
1156#[derive(Debug, Clone, PartialEq, Eq)]
1157pub struct ExitReport {
1158    pub kind: ExitKind,
1159    pub code: Option<i32>,
1160    pub signal: Option<i32>,
1161    pub at_ms: u64,
1162}
1163
1164/// Point-in-time module status answerable by subc without forwarding to the
1165/// module process.
1166#[derive(Debug, Clone, PartialEq)]
1167pub struct ModuleStatus {
1168    pub module_id: String,
1169    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1170    pub waiting_for_previous_pid: Option<u32>,
1171    /// Why the previous instance still prevents a new start.
1172    pub waiting_for_previous_detail: Option<String>,
1173    /// The daemon's Windows per-spawn record: `pipe` after the process it
1174    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1175    /// List/status reports use this record, not the source the module sends
1176    /// in its registration message's manifest.provenance.launch_nonce_source.
1177    pub launch_nonce_source: Option<String>,
1178    pub state: ModuleState,
1179    pub enabled: bool,
1180    pub process_alive: bool,
1181    pub registration_active: bool,
1182    /// The module's declared wire protocol, carried beside `live` because it is
1183    /// what makes `live` readable: the two fields answer one question together.
1184    /// While a process is alive this is its launch declaration, not a later
1185    /// pending-reload edit. When down it is the configured next launch protocol.
1186    pub protocol: ModuleProtocol,
1187    /// Whether the module is serving, under the strongest definition the daemon
1188    /// can assert for its protocol.
1189    ///
1190    /// A subc module must also be REGISTERED: its process being alive says
1191    /// nothing about whether it can take a request. A `protocol: "none"` module
1192    /// never registers, so that term is dropped and this falls back to "enabled,
1193    /// running, and the process the daemon launched is alive" -- which is all
1194    /// the daemon observes about a process that speaks no subc wire. It stays a
1195    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1196    /// rather than printing it bare.
1197    pub live: bool,
1198    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1199    /// restarts have already released their slot, so this count can go down
1200    /// without anybody touching the module.
1201    pub restart_count: u32,
1202    /// Replacement processes spawned over this module's entire supervisor lifetime;
1203    /// unlike `restart_count`, this value is never reset by an operator action
1204    /// and never falls out of a window.
1205    pub lifetime_restarts: u32,
1206    pub spawn_generation: u64,
1207    /// The budget `restart_count` is spent against. Carried alongside the count
1208    /// because the count alone does not say how close the module is to being
1209    /// disabled, and reporting one without the other is what makes an
1210    /// about-to-be-retired module look ordinary.
1211    pub max_restarts: u32,
1212    /// The span `restart_count` is counted over. Carried with the pair above for
1213    /// the same reason they are carried together: "2 of 3" means one thing for a
1214    /// ten-minute window and something else entirely for a lifetime.
1215    pub restart_window: Duration,
1216    /// Effective drain and restart timing policy used by this running module.
1217    /// These values are carried together with the restart budget so status
1218    /// readers can compare configured intent with what the supervisor applied.
1219    pub drain_timeout: Duration,
1220    pub restart_backoff: Duration,
1221    pub restart_max_backoff: Duration,
1222    /// The module's process. On macOS this stays absent while the `ck-subc`
1223    /// launch trampoline is still running in that pid, and appears once the
1224    /// exec acknowledgement confirms the module image has replaced it. Launch
1225    /// time and the supervisor's own hold on the process are unaffected.
1226    pub pid: Option<u32>,
1227    pub spawned_at_ms: Option<u64>,
1228    pub spawned_from: Option<PathBuf>,
1229    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1230    pub process_start_time: Option<u64>,
1231    pub last_exit: Option<ExitReport>,
1232    pub health: ModuleHealthStatus,
1233}
1234
1235#[cfg(test)]
1236#[derive(Debug, Clone, Copy, PartialEq)]
1237struct ActorSelectCheckpoint {
1238    generation: u64,
1239    turn: u64,
1240    registered_connection: Option<ConnectionId>,
1241    next_probe_at: Option<Instant>,
1242    wake_after: Option<Duration>,
1243}
1244
1245#[derive(Debug, Clone, PartialEq)]
1246struct SupervisorSnapshot {
1247    failed_retry: FailedRetry,
1248    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1249    #[cfg(windows)]
1250    pipe_consumed: Option<Arc<OnceLock<()>>>,
1251    /// Counts running-child loop turns, not executor polls of a parked wait.
1252    #[cfg(test)]
1253    actor_turns: u64,
1254    /// Running select after exec confirmation and with no unconsumed registry
1255    /// event. Recording a spawn or starting a loop turn is not this barrier.
1256    #[cfg(test)]
1257    actor_select: Option<ActorSelectCheckpoint>,
1258    state: ModuleState,
1259    enabled: bool,
1260    process_alive: bool,
1261    spawned_protocol: Option<ModuleProtocol>,
1262    spawn_failure: Option<String>,
1263    waiting_for_previous_pid: Option<u32>,
1264    waiting_for_previous_detail: Option<String>,
1265    /// When each crash restart was spent, oldest first. This IS the crash
1266    /// budget: its in-window length is the count an operator sees and the count
1267    /// the restart decision is made against, so there is no second counter that
1268    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1269    /// operator actions that used to zero the old lifetime counter.
1270    crash_restarts: VecDeque<Instant>,
1271    lifetime_restarts: u32,
1272    /// Successful child spawns in this daemon incarnation.
1273    ///
1274    /// `lifetime_restarts` was considered and rejected: it starts at zero
1275    /// (line 640), successful initial/operator spawns in `set_running` do not
1276    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1277    /// increments before a successful replacement exists (lines 604, 3846,
1278    /// and 3921), so a failed spawn can consume it. This counter moves only
1279    /// when a live PID is accepted below.
1280    spawn_generation: u64,
1281    pid: Option<u32>,
1282    #[cfg(target_os = "macos")]
1283    report_ready: Option<Arc<OnceLock<()>>>,
1284    /// Last reaped child, retained after current process facts are cleared.
1285    reaped_pid: Option<u32>,
1286    /// Whether a replacement has already been scheduled for the current process.
1287    respawn_pending: bool,
1288    /// A second restart is waiting for the replacement already scheduled.
1289    coalesced_restart_pending: bool,
1290    spawned_at_ms: Option<u64>,
1291    spawned_from: Option<PathBuf>,
1292    launch_spec_digest: Option<LaunchSpecDigest>,
1293    spawned_file_identity: Option<SpawnedFileIdentity>,
1294    process_start_time: Option<u64>,
1295    deliberate_severance: Option<ProcessIdentity>,
1296    last_exit: Option<ExitReport>,
1297    /// Diagnostic attached to the next drain's terminal record, if any.
1298    drain_disposition_detail: Option<String>,
1299    health: ModuleHealthStatus,
1300    /// Whether the current process was started as a swap candidate and so
1301    /// lives in the module's alternate cgroup. The next swap's candidate takes
1302    /// the other one, so the two processes of a swap never share a cgroup. A
1303    /// plain spawn always uses the primary cgroup.
1304    in_alternate_slot: bool,
1305    /// Whether the current `Draining` state ends in a replacement process
1306    /// (restart, reload, health restart) rather than a stop. Only meaningful
1307    /// while `state` is `Draining`; every entry into that state rewrites it.
1308    /// It is what lets route.open answer the retryable `module_reloading` to a
1309    /// consumer that reaches a still-registered process mid-restart, instead of
1310    /// the `supervisor_not_live` a stop or disable deserves.
1311    draining_to_replace: bool,
1312    /// Whether a configuration update has been applied since the current
1313    /// process was spawned, so that process runs an older spec than the one
1314    /// the supervisor now holds. A queued restart is only coalesced into a
1315    /// fresher process when this is false: a restart requested to pick up a
1316    /// new configuration must not be satisfied by a process that predates it.
1317    configuration_updated_since_spawn: bool,
1318}
1319
1320impl SupervisorSnapshot {
1321    /// The pid that status, provenance and resource readings may report. While
1322    /// the launch trampoline still runs in the pid, reading its executable or
1323    /// resource use would describe `ck-subc`, not the module, so none is
1324    /// reported until the exec acknowledgement confirms the module image.
1325    fn reported_pid(&self) -> Option<u32> {
1326        #[cfg(target_os = "macos")]
1327        if self
1328            .report_ready
1329            .as_ref()
1330            .is_some_and(|ready| ready.get().is_none())
1331        {
1332            return None;
1333        }
1334        self.pid
1335    }
1336
1337    fn starting() -> Self {
1338        Self::new(ModuleState::Starting, true)
1339    }
1340
1341    fn disabled() -> Self {
1342        Self::new(ModuleState::Disabled, false)
1343    }
1344
1345    /// Crash restarts still inside `window`, having dropped the ones that are
1346    /// not. Pruning on read is what makes the budget a rate: an instant older
1347    /// than the window stops holding a slot the moment anybody counts.
1348    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1349        while let Some(oldest) = self.crash_restarts.front() {
1350            if now.duration_since(*oldest) > window {
1351                self.crash_restarts.pop_front();
1352            } else {
1353                break;
1354            }
1355        }
1356        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1357    }
1358
1359    /// Spend one unit of the crash budget and record the restart in the ledger.
1360    ///
1361    /// The ring is bounded by the cap because more than `max_restarts` in-window
1362    /// instants can never be reached (the caller refuses the restart first), so
1363    /// anything beyond that is an unbounded queue waiting to happen.
1364    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1365        self.crash_restarts.push_back(now);
1366        while self.crash_restarts.len() > policy.max_restarts as usize {
1367            self.crash_restarts.pop_front();
1368        }
1369        self.lifetime_restarts += 1;
1370    }
1371
1372    /// Reserve one crash-restart slot and calculate the delay before respawning.
1373    /// The count is captured before recording this restart, so the first retry
1374    /// uses the base delay and each later in-window retry escalates once.
1375    fn next_crash_restart(
1376        &mut self,
1377        policy: &RestartPolicy,
1378        now: Instant,
1379    ) -> Option<CrashRestartSchedule> {
1380        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1381        if restart_in_window >= policy.max_restarts {
1382            return None;
1383        }
1384        self.record_crash_restart(policy, now);
1385        Some(CrashRestartSchedule {
1386            restart_in_window,
1387            delay: policy.delay_for_restart(restart_in_window),
1388        })
1389    }
1390
1391    /// Give the module its full budget back, as an operator restart, reload, or
1392    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1393    /// ledger of what actually happened, and an operator action does not unmake
1394    /// the crashes.
1395    fn clear_crash_restarts(&mut self) {
1396        self.crash_restarts.clear();
1397    }
1398
1399    fn new(state: ModuleState, enabled: bool) -> Self {
1400        Self {
1401            failed_retry: FailedRetry::default(),
1402            exec_outcome: None,
1403            #[cfg(test)]
1404            actor_turns: 0,
1405            #[cfg(test)]
1406            actor_select: None,
1407            state,
1408            enabled,
1409            process_alive: false,
1410            #[cfg(windows)]
1411            pipe_consumed: None,
1412            spawned_protocol: None,
1413            spawn_failure: None,
1414            waiting_for_previous_pid: None,
1415            waiting_for_previous_detail: None,
1416            crash_restarts: VecDeque::new(),
1417            lifetime_restarts: 0,
1418            spawn_generation: 0,
1419            pid: None,
1420            #[cfg(target_os = "macos")]
1421            report_ready: None,
1422            reaped_pid: None,
1423            respawn_pending: false,
1424            coalesced_restart_pending: false,
1425            spawned_at_ms: None,
1426            spawned_from: None,
1427            launch_spec_digest: None,
1428            spawned_file_identity: None,
1429            process_start_time: None,
1430            deliberate_severance: None,
1431            last_exit: None,
1432            drain_disposition_detail: None,
1433            health: ModuleHealthStatus::default(),
1434            in_alternate_slot: false,
1435            draining_to_replace: false,
1436            configuration_updated_since_spawn: false,
1437        }
1438    }
1439}
1440
1441type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1442
1443type SpawnSubscriberKey = (ConnectionId, u64);
1444
1445#[derive(Debug)]
1446struct SpawnSubscriber {
1447    version: u8,
1448    frames: mpsc::Sender<Frame>,
1449    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1450    /// from which event. The full frame channel cannot carry that news, so it
1451    /// travels beside it; see `SpawnEventFeed::subscribe`.
1452    lagged: Option<oneshot::Sender<SpawnCursor>>,
1453}
1454
1455#[derive(Debug)]
1456struct SpawnEventState {
1457    daemon_incarnation: String,
1458    seq: u64,
1459    capacity: usize,
1460    live: HashMap<String, LiveSpawn>,
1461    generations: HashMap<String, u64>,
1462    events: VecDeque<SpawnEvent>,
1463    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1464}
1465
1466impl Default for SpawnEventState {
1467    fn default() -> Self {
1468        Self {
1469            daemon_incarnation: "unconfigured".to_string(),
1470            seq: 0,
1471            capacity: SPAWN_EVENT_RING_CAPACITY,
1472            live: HashMap::new(),
1473            generations: HashMap::new(),
1474            events: VecDeque::new(),
1475            subscribers: HashMap::new(),
1476        }
1477    }
1478}
1479
1480#[derive(Debug, Clone, Default)]
1481struct SpawnEventFeed(
1482    Arc<Mutex<SpawnEventState>>,
1483    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1484);
1485
1486#[derive(Debug, Clone, PartialEq, Eq)]
1487pub(crate) enum SpawnSubscribeRefusal {
1488    ForeignIncarnation { current: String },
1489    TooOld { oldest: SpawnCursor },
1490    Frame(String),
1491}
1492
1493impl SpawnEventFeed {
1494    fn configure_incarnation(&self, daemon_incarnation: String) {
1495        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1496        state.daemon_incarnation = daemon_incarnation;
1497        state.seq = 0;
1498        state.live.clear();
1499        state.generations.clear();
1500        state.events.clear();
1501        state.subscribers.clear();
1502    }
1503
1504    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1505        SpawnCursor {
1506            daemon_incarnation: state.daemon_incarnation.clone(),
1507            seq: state.seq,
1508        }
1509    }
1510
1511    fn snapshot(&self) -> SpawnSnapshot {
1512        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1513        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1514        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1515        SpawnSnapshot {
1516            cursor: Self::cursor(&state),
1517            ring_bound: state.capacity as u64,
1518            live,
1519        }
1520    }
1521
1522    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1523        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1524        let generation = state
1525            .generations
1526            .get(module_id)
1527            .copied()
1528            .unwrap_or(0)
1529            .checked_add(1)
1530            .expect("spawn generation exhausted");
1531        state.generations.insert(module_id.to_string(), generation);
1532        let live = LiveSpawn {
1533            module_id: module_id.to_string(),
1534            spawn_generation: generation,
1535            pid,
1536            spawned_at_ms,
1537        };
1538        state.live.insert(module_id.to_string(), live);
1539        Self::emit_locked(
1540            &mut state,
1541            SpawnEventKind::Spawned,
1542            module_id.to_string(),
1543            generation,
1544            pid,
1545            None,
1546            None,
1547        );
1548        generation
1549    }
1550
1551    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1552        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1553        let Some(live) = state.live.remove(module_id) else {
1554            warn!(
1555                module_id,
1556                "terminal record had no live spawn event identity"
1557            );
1558            return;
1559        };
1560        Self::emit_locked(
1561            &mut state,
1562            SpawnEventKind::Exited,
1563            module_id.to_string(),
1564            live.spawn_generation,
1565            live.pid,
1566            exit_code,
1567            exit_signal,
1568        );
1569    }
1570
1571    /// Report the exit of a process that a swap has already replaced.
1572    ///
1573    /// `emit_exited` removes the module's live entry, which after a swap's
1574    /// cutover describes the promoted candidate, not the old process now
1575    /// exiting. This emits the old generation's exit and leaves the live entry
1576    /// alone unless it still names that generation.
1577    fn emit_superseded_exited(
1578        &self,
1579        module_id: &str,
1580        spawn_generation: u64,
1581        pid: u32,
1582        exit_code: Option<i32>,
1583        exit_signal: Option<i32>,
1584    ) {
1585        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1586        if state
1587            .live
1588            .get(module_id)
1589            .is_some_and(|live| live.spawn_generation == spawn_generation)
1590        {
1591            state.live.remove(module_id);
1592        }
1593        Self::emit_locked(
1594            &mut state,
1595            SpawnEventKind::Exited,
1596            module_id.to_string(),
1597            spawn_generation,
1598            pid,
1599            exit_code,
1600            exit_signal,
1601        );
1602    }
1603
1604    #[allow(clippy::too_many_arguments)]
1605    fn emit_locked(
1606        state: &mut SpawnEventState,
1607        kind: SpawnEventKind,
1608        module_id: String,
1609        spawn_generation: u64,
1610        pid: u32,
1611        exit_code: Option<i32>,
1612        exit_signal: Option<i32>,
1613    ) {
1614        state.seq = state
1615            .seq
1616            .checked_add(1)
1617            .expect("spawn event sequence exhausted");
1618        let event = SpawnEvent {
1619            cursor: Self::cursor(state),
1620            kind,
1621            module_id,
1622            spawn_generation,
1623            pid,
1624            exit_code,
1625            exit_signal,
1626        };
1627        state.events.push_back(event.clone());
1628        while state.events.len() > state.capacity {
1629            state.events.pop_front();
1630        }
1631        let body = match serde_json::to_vec(&event) {
1632            Ok(body) => body,
1633            Err(error) => {
1634                error!(%error, "failed to serialize supervisor spawn event");
1635                return;
1636            }
1637        };
1638        state.subscribers.retain(|(connection_id, corr), subscriber| {
1639            let frame = Frame::build_with_version(
1640                subscriber.version,
1641                FrameType::StreamData,
1642                control_flags(),
1643                0,
1644                0,
1645                *corr,
1646                body.clone(),
1647            );
1648            match frame {
1649                Ok(frame) => {
1650                    if subscriber.frames.try_send(frame).is_ok() {
1651                        true
1652                    } else {
1653                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1654                        if let Some(lagged) = subscriber.lagged.take() {
1655                            let _ = lagged.send(event.cursor.clone());
1656                        }
1657                        false
1658                    }
1659                }
1660                Err(error) => {
1661                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1662                    false
1663                }
1664            }
1665        });
1666    }
1667
1668    fn subscribe(
1669        &self,
1670        connection_id: ConnectionId,
1671        corr: u64,
1672        version: u8,
1673        since: Option<SpawnCursor>,
1674        sink: FrameSink,
1675    ) -> Result<(), SpawnSubscribeRefusal> {
1676        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1677        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1678        {
1679            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1680            let replay = if let Some(since) = since {
1681                if since.daemon_incarnation != state.daemon_incarnation {
1682                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1683                        current: state.daemon_incarnation.clone(),
1684                    });
1685                }
1686                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1687                    if since.seq < oldest.seq.saturating_sub(1) {
1688                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1689                    }
1690                }
1691                state
1692                    .events
1693                    .iter()
1694                    .filter(|event| event.cursor.seq > since.seq)
1695                    .cloned()
1696                    .collect::<Vec<_>>()
1697            } else {
1698                Vec::new()
1699            };
1700            for event in replay {
1701                let body = serde_json::to_vec(&event)
1702                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1703                let frame = Frame::build_with_version(
1704                    version,
1705                    FrameType::StreamData,
1706                    control_flags(),
1707                    0,
1708                    0,
1709                    corr,
1710                    body,
1711                )
1712                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1713                frames
1714                    .try_send(frame)
1715                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716            }
1717            state.subscribers.insert(
1718                (connection_id, corr),
1719                SpawnSubscriber {
1720                    version,
1721                    frames,
1722                    lagged: Some(lagged),
1723                },
1724            );
1725        }
1726        // The lagged terminal is sent here, by the forwarder, rather than by
1727        // the emitter: at the moment of the drop the subscriber's own channel
1728        // is full, and writing to the connection sink directly from the emitter
1729        // would put the Error AHEAD of the events still queued in that channel
1730        // (and the emitter holds the feed lock, so it cannot await the sink).
1731        // Dropping the subscriber drops the only sender, so `recv` drains every
1732        // queued event and then returns `None`; only then is the Error sent, so
1733        // the client sees each event it can keep, then the reason it was cut.
1734        // Cancel and connection removal drop the oneshot unsent, so they end
1735        // the stream with no Error.
1736        tokio::spawn(async move {
1737            while let Some(frame) = receiver.recv().await {
1738                if sink.send(frame).await.is_err() {
1739                    return;
1740                }
1741            }
1742            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1743                return;
1744            };
1745            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1746                Ok(frame) => {
1747                    let _ = sink.send(frame).await;
1748                }
1749                Err(error) => {
1750                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1751                }
1752            }
1753        });
1754        Ok(())
1755    }
1756
1757    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1758        let Some(subscriber) = self
1759            .0
1760            .lock()
1761            .unwrap_or_else(|p| p.into_inner())
1762            .subscribers
1763            .remove(&(connection_id, corr))
1764        else {
1765            return false;
1766        };
1767        if let Ok(frame) = Frame::build_with_version(
1768            subscriber.version,
1769            FrameType::StreamEnd,
1770            control_flags(),
1771            0,
1772            0,
1773            corr,
1774            Vec::new(),
1775        ) {
1776            tokio::spawn(async move {
1777                let _ = subscriber.frames.send(frame).await;
1778            });
1779        }
1780        true
1781    }
1782
1783    fn remove_connection(&self, connection_id: ConnectionId) {
1784        self.0
1785            .lock()
1786            .unwrap_or_else(|p| p.into_inner())
1787            .subscribers
1788            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1789    }
1790
1791    #[cfg(any(test, feature = "test-support"))]
1792    fn set_capacity(&self, capacity: usize) {
1793        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1794    }
1795
1796    #[cfg(any(test, feature = "test-support"))]
1797    fn subscriber_count(&self) -> usize {
1798        self.0
1799            .lock()
1800            .unwrap_or_else(|p| p.into_inner())
1801            .subscribers
1802            .len()
1803    }
1804}
1805
1806/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1807/// The terminal Error a lagged spawn subscriber receives after its queued events.
1808fn spawn_subscriber_lagged_frame(
1809    version: u8,
1810    corr: u64,
1811    first_undelivered: SpawnCursor,
1812) -> Result<Frame, String> {
1813    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1814        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1815        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1816            .to_string(),
1817        detail: Some(serde_json::json!({
1818            "first_undelivered_cursor": first_undelivered
1819        })),
1820    })
1821    .map_err(|error| error.to_string())?;
1822    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1823        .map_err(|error| error.to_string())
1824}
1825
1826pub trait ModuleProcessLiveness: Send + Sync {
1827    fn process_live(&self, module_id: &str) -> Option<bool>;
1828
1829    /// Whether the supervisor is replacing this module's process right now: an
1830    /// operator restart or reload, a health restart, or a crash respawn whose
1831    /// backoff is running. A module in that state is not live, but a consumer
1832    /// refused now should retry shortly rather than treat the target as gone.
1833    /// Stopped, failed, and disabled modules are not replacing.
1834    fn process_replacing(&self, _module_id: &str) -> bool {
1835        false
1836    }
1837}
1838
1839/// Shared process-liveness registry keyed by supervised `module_id`.
1840#[derive(Debug, Clone, Default)]
1841pub struct SupervisorProcessLiveness {
1842    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1843}
1844
1845impl SupervisorProcessLiveness {
1846    pub fn new() -> Self {
1847        Self::default()
1848    }
1849
1850    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1851        let mut snapshots = self
1852            .snapshots
1853            .lock()
1854            .unwrap_or_else(|poisoned| poisoned.into_inner());
1855        snapshots.insert(module_id, snapshot);
1856    }
1857
1858    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1859        let mut snapshots = self
1860            .snapshots
1861            .lock()
1862            .unwrap_or_else(|poisoned| poisoned.into_inner());
1863        let is_current = snapshots
1864            .get(module_id)
1865            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1866            .unwrap_or(false);
1867        if is_current {
1868            snapshots.remove(module_id);
1869        }
1870    }
1871}
1872
1873impl ModuleProcessLiveness for SupervisorProcessLiveness {
1874    fn process_live(&self, module_id: &str) -> Option<bool> {
1875        let snapshot = {
1876            let snapshots = self
1877                .snapshots
1878                .lock()
1879                .unwrap_or_else(|poisoned| poisoned.into_inner());
1880            snapshots.get(module_id).cloned()
1881        }?;
1882        let snapshot = snapshot
1883            .lock()
1884            .unwrap_or_else(|poisoned| poisoned.into_inner());
1885        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1886    }
1887
1888    fn process_replacing(&self, module_id: &str) -> bool {
1889        let Some(snapshot) = self
1890            .snapshots
1891            .lock()
1892            .unwrap_or_else(|poisoned| poisoned.into_inner())
1893            .get(module_id)
1894            .cloned()
1895        else {
1896            return false;
1897        };
1898        let snapshot = snapshot
1899            .lock()
1900            .unwrap_or_else(|poisoned| poisoned.into_inner());
1901        snapshot.enabled
1902            && match snapshot.state {
1903                ModuleState::Restarting => true,
1904                ModuleState::Draining => snapshot.draining_to_replace,
1905                ModuleState::Starting
1906                | ModuleState::Running
1907                | ModuleState::Unresponsive
1908                | ModuleState::Stopped
1909                | ModuleState::Failed
1910                | ModuleState::Disabled => false,
1911            }
1912    }
1913}
1914
1915#[cfg(test)]
1916#[derive(Debug, Default)]
1917struct ReloadExitRecordGate {
1918    reached: tokio::sync::Notify,
1919    resume: tokio::sync::Notify,
1920}
1921
1922#[derive(Debug, Clone, Copy)]
1923enum RespawnKind {
1924    Spawn,
1925    Reload,
1926}
1927
1928#[derive(Debug, Clone, Copy)]
1929struct PendingRespawn {
1930    deadline: Instant,
1931    kind: RespawnKind,
1932    daemon_initiated: bool,
1933}
1934
1935type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1936
1937#[derive(Debug, Clone)]
1938struct SupervisorRuntimeConfig {
1939    registry: Arc<Registry>,
1940    /// Retained across an operation's deferred backoff, until its child starts.
1941    recovery_is_daemon: bool,
1942    start_requester: Option<Principal>,
1943    pending_activity_restart: bool,
1944    provenance_probe: ExecutableIdentityProbe,
1945    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1946    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1947    /// A reload acknowledges completion only after its replacement registers.
1948    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1949    restart_policy: RestartPolicy,
1950    register_timeout: Duration,
1951    /// This module's RESOLVED drain budget: per-module config when present,
1952    /// else `default_drain_timeout`.
1953    drain_timeout: Duration,
1954    /// Shared with the status handle so the attested value changes atomically
1955    /// when a rescan updates the running drain policy.
1956    effective_drain_timeout: Arc<Mutex<Duration>>,
1957    /// The supervisor-wide fallback, kept so a configuration update that
1958    /// REMOVES the per-module override can re-resolve to it.
1959    default_drain_timeout: Duration,
1960    health: HealthConfig,
1961    connection_file_path: Option<PathBuf>,
1962    capture_logs_dir: Option<PathBuf>,
1963    forwarding: Option<Arc<ForwardingTable>>,
1964    /// The shared handle, so every spawn path (initial, restart, reload) records the
1965    /// reserved-module launch nonce the HELLO verifier checks against.
1966    supervisor_handle: Option<SupervisorHandle>,
1967    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1968    /// status queries.
1969    ///
1970    /// One ring per module, held across every respawn. The lines explaining an exit
1971    /// are written BEFORE that exit, so a ring recreated per process would be empty
1972    /// exactly when it is asked for.
1973    stderr_ring: Arc<Mutex<StderrRing>>,
1974    terminal_ring: Arc<Mutex<TerminalRing>>,
1975    spawn_events: SpawnEventFeed,
1976    child_roster: ChildRoster,
1977    #[cfg(target_os = "linux")]
1978    cgroup_placement: Option<subc_cgroup::Placement>,
1979    #[cfg(test)]
1980    test_seed_stale_facts_before_enable_spawn: bool,
1981    #[cfg(test)]
1982    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1983}
1984
1985impl SupervisorRuntimeConfig {
1986    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1987        FailureContext {
1988            daemon_initiated,
1989            shutdown: self.child_roster.shutdown_flag(),
1990        }
1991    }
1992}
1993
1994#[derive(Debug, Clone, PartialEq, Eq)]
1995struct SupervisedConfiguration {
1996    spec: ModuleSpec,
1997    health: HealthConfig,
1998}
1999
2000/// Shared daemon lookup table for supervised module handles.
2001///
2002/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2003/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2004/// launch nonces recorded at spawn are checked by the same daemon instance.
2005#[derive(Debug, Clone, Default)]
2006pub struct SupervisorHandle {
2007    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2008    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2009    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2010    /// Module ids the supervisor has taken on. An id is added BEFORE the
2011    /// module's first process is spawned and removed only when the module
2012    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2013    /// the keys of `modules`.
2014    ///
2015    /// `modules` cannot answer "is this module configured?" on its own: a
2016    /// [`SupervisedModule`] only exists once its process has been spawned, and
2017    /// a fast child can connect, register, sync its scopes and ask about them
2018    /// before the supervisor has inserted it. Answering "not configured" in that
2019    /// gap makes scope admission refuse with the terminal "will never sync"
2020    /// instead of the retryable "has not synced yet".
2021    configured_ids: Arc<Mutex<HashSet<String>>>,
2022    spawn_events: SpawnEventFeed,
2023    /// The current expected launch nonce for each reserved module_id. Set when the
2024    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2025    /// non-reserved module never has an entry here and is never nonce-checked.
2026    /// Reserved module ids and the nonce that authorizes their next HELLO.
2027    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2028    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2029    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2030    /// had NO entry and admitted anyone: the reservation protected the nonce
2031    /// holder, not the NAME (found live by CKCRED's canary probe registering
2032    /// against a reserved scratch id).
2033    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2034    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2035    ///
2036    /// This is deliberately in-memory only: subc is state-free across daemon
2037    /// restarts, and the tombstone only explains the hours-after-removal window
2038    /// while this executing daemon is still alive. Do not persist it in a store.
2039    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2040    /// The current launch nonce for every supervised spawn. This is separate from
2041    /// reserved_nonces because consumer route.open attestation applies to all spawned
2042    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2043    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2044    /// Reserved namespace prefixes mapped to the supervised owner module whose
2045    /// current spawn nonce authorizes HELLO claims below the prefix.
2046    ///
2047    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2048    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2049    /// accidental collisions and lower-trust processes from squatting protected
2050    /// namespaces.
2051    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2052    /// Blue/green swaps in progress, by module id. An entry exists from just
2053    /// before the candidate process is spawned until the swap has failed, or
2054    /// has cut over and the old process is gone. While it exists, HELLO for the
2055    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2056    /// consumer attestation accepts both processes' nonces.
2057    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2058    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2059    promotion_observer: PromotionObserverSlot,
2060    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2061    /// this daemon-wide ordering, a rescan could retire or update a module while a
2062    /// concurrent reload still held its old handle and launch specification.
2063    operation_lock: Arc<AsyncMutex<()>>,
2064}
2065
2066/// Told when a swap has promoted its candidate to be the module's active
2067/// registration.
2068///
2069/// An ordinary HELLO runs the control plane's registration side effects (the
2070/// capability cache, the deny census, the requirement recompute) as it
2071/// registers. A swap candidate's HELLO does not, because it is not routable;
2072/// promotion is when those must run instead, and promotion happens in the
2073/// supervisor, which has no other way into the control handler.
2074pub(crate) trait SwapPromotionObserver: Send + Sync {
2075    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2076}
2077
2078/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2079/// control handler) owns this handle, so a strong reference back would be a
2080/// cycle that keeps both alive.
2081#[derive(Clone, Default)]
2082struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2083
2084impl fmt::Debug for PromotionObserverSlot {
2085    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2086        f.write_str("PromotionObserverSlot")
2087    }
2088}
2089
2090/// The nonces of one open swap.
2091#[derive(Debug, Clone)]
2092struct OpenSwap {
2093    /// The launch nonce minted for the candidate process. It is the swap
2094    /// token: the only thing that admits a HELLO into the candidate slot.
2095    candidate_nonce: String,
2096    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2097    /// here because cutover moves the module's recorded spawn nonce to the
2098    /// candidate while the incumbent is still draining and its consumers are
2099    /// still attesting with this one.
2100    incumbent_nonce: Option<String>,
2101    /// Set once a HELLO has been admitted with the swap token, so the token
2102    /// admits one registration and cannot be replayed after cutover empties
2103    /// the candidate slot.
2104    candidate_admitted: bool,
2105}
2106
2107/// What the swap gate says about a HELLO. See
2108/// [`SupervisorHandle::swap_hello_admission`].
2109#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2110pub(crate) enum SwapHelloAdmission {
2111    /// No swap is open for the id (or the HELLO carries the incumbent's own
2112    /// nonce); the ordinary gates decide.
2113    NotSwapping,
2114    /// The HELLO carries the swap token: register it into the candidate slot.
2115    Candidate,
2116    /// A swap is open and the HELLO carries a nonce the supervisor did not
2117    /// mint for this id, no nonce, or a token already used.
2118    Refused,
2119}
2120
2121#[derive(Debug, Clone, PartialEq, Eq)]
2122pub(crate) enum ReservedHelloRejection {
2123    Exact {
2124        module_id: String,
2125    },
2126    Prefix {
2127        prefix: String,
2128        owner_module_id: String,
2129    },
2130}
2131
2132impl SupervisorHandle {
2133    pub fn new() -> Self {
2134        Self::default()
2135    }
2136
2137    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2138        self.spawn_events.snapshot()
2139    }
2140
2141    pub(crate) fn subscribe_spawns(
2142        &self,
2143        connection_id: ConnectionId,
2144        corr: u64,
2145        version: u8,
2146        since: Option<SpawnCursor>,
2147        sink: FrameSink,
2148    ) -> Result<(), SpawnSubscribeRefusal> {
2149        self.spawn_events
2150            .subscribe(connection_id, corr, version, since, sink)
2151    }
2152
2153    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2154        self.spawn_events.cancel(connection_id, corr)
2155    }
2156
2157    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2158        self.spawn_events.remove_connection(connection_id);
2159    }
2160
2161    #[cfg(any(test, feature = "test-support"))]
2162    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2163        assert!(capacity > 0, "spawn event capacity must be non-zero");
2164        self.spawn_events.set_capacity(capacity);
2165    }
2166
2167    #[cfg(any(test, feature = "test-support"))]
2168    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2169        self.spawn_events.subscriber_count()
2170    }
2171
2172    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2173    /// a respawn invalidates stale consumer identities.
2174    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2175        self.spawn_nonces
2176            .lock()
2177            .unwrap_or_else(|poisoned| poisoned.into_inner())
2178            .insert(module_id.to_string(), nonce);
2179    }
2180
2181    /// Record the launch nonce expected from the next HELLO for a reserved module,
2182    /// replacing any prior nonce (a respawn invalidates the previous one).
2183    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2184        self.reserved_nonces
2185            .lock()
2186            .unwrap_or_else(|poisoned| poisoned.into_inner())
2187            .insert(module_id.to_string(), Some(nonce));
2188    }
2189
2190    /// Record namespace prefixes owned by a supervised module.
2191    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2192        let mut owners = self
2193            .reserved_prefix_owners
2194            .lock()
2195            .unwrap_or_else(|poisoned| poisoned.into_inner());
2196        owners.retain(|_, owner| owner != owner_module_id);
2197        for prefix in prefixes {
2198            owners.insert(prefix.clone(), owner_module_id.to_string());
2199        }
2200    }
2201
2202    /// The launch nonce most recently minted for a module's spawn, if any.
2203    #[cfg(test)]
2204    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2205        self.spawn_nonces
2206            .lock()
2207            .unwrap_or_else(|poisoned| poisoned.into_inner())
2208            .get(module_id)
2209            .cloned()
2210    }
2211
2212    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2213        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2214        let spawn_nonce = self
2215            .spawn_nonces
2216            .lock()
2217            .unwrap_or_else(|poisoned| poisoned.into_inner())
2218            .get(&spec.module_id)
2219            .cloned();
2220        let mut reserved_nonces = self
2221            .reserved_nonces
2222            .lock()
2223            .unwrap_or_else(|poisoned| poisoned.into_inner());
2224        if spec.reserved {
2225            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2226            // reserved name whose module has never spawned has no legitimate
2227            // holder, and the entry's absence is what used to leave the name
2228            // open to the first claimant.
2229            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2230        }
2231        drop(reserved_nonces);
2232        // A later unreserved declaration must not silently unreserve an id that
2233        // was retained after its reserved configuration was removed. The explicit
2234        // release ceremony is the only operation that retires that gate.
2235        self.removal_tombstones
2236            .lock()
2237            .unwrap_or_else(|poisoned| poisoned.into_inner())
2238            .remove(&spec.module_id);
2239    }
2240
2241    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2242    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2243    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2244    /// with no matching prefix are always authorized.
2245    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2246        self.reserved_hello_verification(module_id, presented)
2247            .is_ok()
2248    }
2249
2250    /// Whether HELLO passed a nonce check that was actually required by reserved
2251    /// module policy, or the refusal explaining why a required check failed.
2252    pub(crate) fn reserved_hello_verification(
2253        &self,
2254        module_id: &str,
2255        presented: Option<&str>,
2256    ) -> Result<bool, ReservedHelloRejection> {
2257        let nonces = self
2258            .reserved_nonces
2259            .lock()
2260            .unwrap_or_else(|poisoned| poisoned.into_inner());
2261        if let Some(expected) = nonces.get(module_id) {
2262            // `None` = reserved with no legitimate holder: refuse every
2263            // presentation, because no process can hold a nonce that was never
2264            // minted. Only a real minted nonce admits, in constant time.
2265            let authorized = match expected {
2266                Some(expected) => {
2267                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2268                }
2269                None => false,
2270            };
2271            if authorized {
2272                return Ok(true);
2273            }
2274            return Err(ReservedHelloRejection::Exact {
2275                module_id: module_id.to_string(),
2276            });
2277        }
2278        drop(nonces);
2279
2280        let matched_prefix = self
2281            .reserved_prefix_owners
2282            .lock()
2283            .unwrap_or_else(|poisoned| poisoned.into_inner())
2284            .iter()
2285            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2286            .max_by_key(|(prefix, _)| prefix.len())
2287            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2288        let Some((prefix, owner_module_id)) = matched_prefix else {
2289            return Ok(false);
2290        };
2291
2292        let authorized = presented.is_some_and(|presented| {
2293            self.spawn_nonces
2294                .lock()
2295                .unwrap_or_else(|poisoned| poisoned.into_inner())
2296                .get(&owner_module_id)
2297                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2298                // While the owner is being swapped, children started by
2299                // either of its two processes hold that process's nonce.
2300                || self.swap_nonce_matches(&owner_module_id, presented)
2301        });
2302        if authorized {
2303            Ok(true)
2304        } else {
2305            Err(ReservedHelloRejection::Prefix {
2306                prefix,
2307                owner_module_id,
2308            })
2309        }
2310    }
2311
2312    /// Whether a consumer connection proved it came from a daemon-spawned module.
2313    ///
2314    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2315    /// accepted only for module ids the supervisor has spawned.
2316    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2317        if presented.is_empty() {
2318            return false;
2319        }
2320        let nonces = self
2321            .spawn_nonces
2322            .lock()
2323            .unwrap_or_else(|poisoned| poisoned.into_inner());
2324        let current = nonces
2325            .get(module_id)
2326            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2327        drop(nonces);
2328        // During a swap two processes of the module are alive, and a consumer
2329        // started by either one presents that process's nonce. Accepting only
2330        // the recorded one would fail the incumbent's consumers for the whole
2331        // overlap once cutover moves the record to the candidate.
2332        current || self.swap_nonce_matches(module_id, presented)
2333    }
2334
2335    /// Whether `presented` is either nonce of an open swap for `module_id`.
2336    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2337        let swaps = self
2338            .swaps
2339            .lock()
2340            .unwrap_or_else(|poisoned| poisoned.into_inner());
2341        swaps.get(module_id).is_some_and(|swap| {
2342            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2343                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2344                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2345                })
2346        })
2347    }
2348
2349    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2350    /// Called before the candidate process exists.
2351    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2352        let incumbent_nonce = self
2353            .spawn_nonces
2354            .lock()
2355            .unwrap_or_else(|poisoned| poisoned.into_inner())
2356            .get(module_id)
2357            .cloned();
2358        self.swaps
2359            .lock()
2360            .unwrap_or_else(|poisoned| poisoned.into_inner())
2361            .insert(
2362                module_id.to_string(),
2363                OpenSwap {
2364                    candidate_nonce,
2365                    incumbent_nonce,
2366                    candidate_admitted: false,
2367                },
2368            );
2369    }
2370
2371    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2372    /// the module's recorded one.
2373    pub(crate) fn close_swap(&self, module_id: &str) {
2374        self.swaps
2375            .lock()
2376            .unwrap_or_else(|poisoned| poisoned.into_inner())
2377            .remove(module_id);
2378    }
2379
2380    /// Install the observer told about swap promotions, replacing any earlier
2381    /// one.
2382    pub(crate) fn set_swap_promotion_observer(
2383        &self,
2384        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2385    ) {
2386        *self
2387            .promotion_observer
2388            .0
2389            .lock()
2390            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2391    }
2392
2393    /// Tell the installed observer, if it is still alive, that a swap promoted
2394    /// `registration`.
2395    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2396        let observer = self
2397            .promotion_observer
2398            .0
2399            .lock()
2400            .unwrap_or_else(|poisoned| poisoned.into_inner())
2401            .as_ref()
2402            .and_then(std::sync::Weak::upgrade);
2403        if let Some(observer) = observer {
2404            observer.swap_promoted(registration);
2405        }
2406    }
2407
2408    /// Whether a swap is open for `module_id`.
2409    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2410        self.swaps
2411            .lock()
2412            .unwrap_or_else(|poisoned| poisoned.into_inner())
2413            .contains_key(module_id)
2414    }
2415
2416    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2417    /// respawn would, once cutover has made the candidate the module's process.
2418    /// The swap stays open so the incumbent's nonce keeps attesting until the
2419    /// incumbent has drained and exited.
2420    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2421        let candidate_nonce = self
2422            .swaps
2423            .lock()
2424            .unwrap_or_else(|poisoned| poisoned.into_inner())
2425            .get(module_id)
2426            .map(|swap| swap.candidate_nonce.clone());
2427        let Some(nonce) = candidate_nonce else {
2428            return;
2429        };
2430        self.set_spawn_nonce(module_id, nonce.clone());
2431        if reserved {
2432            self.set_reserved_nonce(module_id, nonce);
2433        }
2434    }
2435
2436    /// The swap gate for a HELLO claiming `module_id`.
2437    ///
2438    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2439    /// presents the candidate nonce, which the reserved gate (holding the
2440    /// incumbent's nonce) would refuse as `reserved_module` before swap
2441    /// admission was ever reached. And it applies to unreserved ids too: for an
2442    /// unreserved id the only thing that ever stopped a second process claiming
2443    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2444    /// refusal a swap lifts for its candidate.
2445    ///
2446    /// The incumbent's own nonce falls through to the ordinary gates, which
2447    /// treat it as they always have (a live incumbent is refused as a
2448    /// duplicate). Anything else while a swap is open is refused, including an
2449    /// absent nonce.
2450    pub(crate) fn swap_hello_admission(
2451        &self,
2452        module_id: &str,
2453        presented: Option<&str>,
2454    ) -> SwapHelloAdmission {
2455        let swaps = self
2456            .swaps
2457            .lock()
2458            .unwrap_or_else(|poisoned| poisoned.into_inner());
2459        let Some(swap) = swaps.get(module_id) else {
2460            return SwapHelloAdmission::NotSwapping;
2461        };
2462        let Some(presented) = presented else {
2463            return SwapHelloAdmission::Refused;
2464        };
2465        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2466            return if swap.candidate_admitted {
2467                SwapHelloAdmission::Refused
2468            } else {
2469                SwapHelloAdmission::Candidate
2470            };
2471        }
2472        if swap
2473            .incumbent_nonce
2474            .as_deref()
2475            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2476        {
2477            return SwapHelloAdmission::NotSwapping;
2478        }
2479        SwapHelloAdmission::Refused
2480    }
2481
2482    /// Record that the swap token has registered a candidate, so it admits no
2483    /// second HELLO.
2484    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2485        if let Some(swap) = self
2486            .swaps
2487            .lock()
2488            .unwrap_or_else(|poisoned| poisoned.into_inner())
2489            .get_mut(module_id)
2490        {
2491            swap.candidate_admitted = true;
2492        }
2493    }
2494
2495    /// Test/support lookup for the current launch nonce of a supervised spawn.
2496    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2497        self.spawn_nonces
2498            .lock()
2499            .unwrap_or_else(|poisoned| poisoned.into_inner())
2500            .get(module_id)
2501            .cloned()
2502    }
2503
2504    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2505    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2506        self.reserved_nonces
2507            .lock()
2508            .unwrap_or_else(|poisoned| poisoned.into_inner())
2509            .get(module_id)
2510            .cloned()
2511            .flatten()
2512    }
2513
2514    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2515        // Normally already marked before the process was spawned; marking here
2516        // too keeps `configured_ids` a superset of the roster for any caller
2517        // that inserts a module directly.
2518        self.mark_configured(module.module_id());
2519        let mut modules = self
2520            .modules
2521            .lock()
2522            .unwrap_or_else(|poisoned| poisoned.into_inner());
2523        modules.insert(module.module_id().to_string(), module)
2524    }
2525
2526    /// Record that the supervisor has taken on `module_id`. Called before the
2527    /// module's first process is spawned, so that by the time that process can
2528    /// register, [`Self::is_configured`] already answers true.
2529    fn mark_configured(&self, module_id: &str) {
2530        self.configured_ids
2531            .lock()
2532            .unwrap_or_else(|poisoned| poisoned.into_inner())
2533            .insert(module_id.to_string());
2534    }
2535
2536    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2537    /// before it was ever put on the roster. A module already on the roster
2538    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2539    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2540        let modules = self
2541            .modules
2542            .lock()
2543            .unwrap_or_else(|poisoned| poisoned.into_inner());
2544        if !modules.contains_key(module_id) {
2545            self.configured_ids
2546                .lock()
2547                .unwrap_or_else(|poisoned| poisoned.into_inner())
2548                .remove(module_id);
2549        }
2550    }
2551
2552    /// Whether `module_id` is a module this daemon supervises: on the roster,
2553    /// or about to be (its process is being spawned right now).
2554    ///
2555    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2556    /// for scopes: a supervised module's process can register and sync before
2557    /// [`Self::get`] can return it, and in that window it is still a module
2558    /// that will sync, not one that never will.
2559    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2560        self.configured_ids
2561            .lock()
2562            .unwrap_or_else(|poisoned| poisoned.into_inner())
2563            .contains(module_id)
2564    }
2565
2566    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2567        let modules = self
2568            .modules
2569            .lock()
2570            .unwrap_or_else(|poisoned| poisoned.into_inner());
2571        modules.get(module_id).cloned()
2572    }
2573
2574    pub(crate) fn record_late_health_answer(
2575        &self,
2576        module_id: &str,
2577        latency_ms: u64,
2578    ) -> Result<bool, SuperviseError> {
2579        let Some(module) = self.get(module_id) else {
2580            return Ok(false);
2581        };
2582        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2583            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2584            state.health.last_late_answer_latency_ms = Some(latency_ms);
2585            // A late answer is an answer: the module served the probe, just past
2586            // the deadline. Leaving the miss streak in place while logging
2587            // "proves the module is alive" is how a CPU-starved module that
2588            // answers every probe a few seconds late still marches to the
2589            // threshold and gets killed — the exact kill class `NoAnswer` is
2590            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2591            // is degradation, and degradation reports; it does not restart.
2592            state.health.consecutive_failures = 0;
2593        })?;
2594        Ok(true)
2595    }
2596
2597    /// Arm the one-shot marker for the module process that this caller
2598    /// deliberately initiated severance against. Generic connection teardown
2599    /// must not call this:
2600    /// a surviving process would otherwise retain an exemption for a later
2601    /// genuine crash.
2602    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2603        let Some(module) = self.get(module_id) else {
2604            return Ok(false);
2605        };
2606        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2607        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2608            return Ok(false);
2609        };
2610        drop(snapshot);
2611        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2612    }
2613
2614    pub fn list(&self) -> Vec<SupervisedModule> {
2615        let modules = self
2616            .modules
2617            .lock()
2618            .unwrap_or_else(|poisoned| poisoned.into_inner());
2619        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2620        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2621        modules
2622    }
2623
2624    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2625        self.activity_images
2626            .lock()
2627            .unwrap_or_else(|p| p.into_inner())
2628            .remove(module_id);
2629        self.spawn_nonces
2630            .lock()
2631            .unwrap_or_else(|poisoned| poisoned.into_inner())
2632            .remove(module_id);
2633        self.close_swap(module_id);
2634        let mut reserved_nonces = self
2635            .reserved_nonces
2636            .lock()
2637            .unwrap_or_else(|poisoned| poisoned.into_inner());
2638        if reserved_nonces.contains_key(module_id) {
2639            // The old nonce must die with the removed process, but the exact-id
2640            // gate remains until an operator explicitly releases it.
2641            reserved_nonces.insert(module_id.to_string(), None);
2642        }
2643        drop(reserved_nonces);
2644        self.reserved_prefix_owners
2645            .lock()
2646            .unwrap_or_else(|poisoned| poisoned.into_inner())
2647            .retain(|_, owner| owner != module_id);
2648        let removed = self
2649            .modules
2650            .lock()
2651            .unwrap_or_else(|poisoned| poisoned.into_inner())
2652            .remove(module_id);
2653        self.configured_ids
2654            .lock()
2655            .unwrap_or_else(|poisoned| poisoned.into_inner())
2656            .remove(module_id);
2657        removed
2658    }
2659
2660    /// Remember a module removed by a non-preview rescan so route.open can
2661    /// distinguish that intentional removal from an unknown id.
2662    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2663        self.removal_tombstones
2664            .lock()
2665            .unwrap_or_else(|poisoned| poisoned.into_inner())
2666            .insert(module_id.to_string(), unix_ms_now());
2667    }
2668
2669    /// Return how long ago a rescan removed this module in milliseconds.
2670    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2671        self.removal_tombstones
2672            .lock()
2673            .unwrap_or_else(|poisoned| poisoned.into_inner())
2674            .get(module_id)
2675            .copied()
2676            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2677    }
2678
2679    /// Retire a reserved-id gate only after its module has left supervision.
2680    ///
2681    /// A retained gate has no live nonce (`None`), so releasing any other entry
2682    /// would weaken a currently configured or otherwise active reservation.
2683    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2684        if self.get(module_id).is_some() {
2685            return false;
2686        }
2687        let mut reserved_nonces = self
2688            .reserved_nonces
2689            .lock()
2690            .unwrap_or_else(|poisoned| poisoned.into_inner());
2691        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2692            return false;
2693        }
2694        reserved_nonces.remove(module_id);
2695        true
2696    }
2697
2698    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2699        Arc::clone(&self.operation_lock)
2700    }
2701}
2702
2703/// Process supervisor for subc-owned singleton modules.
2704#[derive(Debug, Clone)]
2705pub struct Supervisor {
2706    registry: Arc<Registry>,
2707    restart_policy: RestartPolicy,
2708    drain_timeout: Duration,
2709    connection_file_path: Option<PathBuf>,
2710    capture_logs_dir: Option<PathBuf>,
2711    forwarding: Option<Arc<ForwardingTable>>,
2712    process_liveness: Arc<SupervisorProcessLiveness>,
2713    supervisor_handle: Option<SupervisorHandle>,
2714    health: HealthConfig,
2715    daemon_start_clock: crate::clock::StartClock,
2716    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2717    spawn_events: SpawnEventFeed,
2718    provenance_probe: ExecutableIdentityProbe,
2719    /// Every process spawned through this supervisor (and its clones) and not
2720    /// yet reaped, so daemon shutdown can end them.
2721    child_roster: ChildRoster,
2722    #[cfg(target_os = "linux")]
2723    cgroup_placement: Option<subc_cgroup::Placement>,
2724    #[cfg(test)]
2725    test_after_first_spawn: AfterFirstSpawnHook,
2726}
2727
2728/// A roster entry whose monitor and first process have not started yet.
2729/// Keeping the command receiver here lets discovery serve status immediately
2730/// while start/stop requests wait until bootstrap has published the connection file.
2731pub(crate) struct RegisteredConfiguredModule {
2732    spec: ModuleSpec,
2733    runtime: SupervisorRuntimeConfig,
2734    module: SupervisedModule,
2735    commands: mpsc::Receiver<SupervisorCommand>,
2736}
2737
2738/// Test-only hook run on the path that takes on a new module, right after its
2739/// first `spawn_child` returns (the process exists and could already be
2740/// registering) and before that process is handed to the module's supervise
2741/// loop and put on the roster. Lets a test observe what a fast child would see
2742/// in that window without racing a real one.
2743#[cfg(test)]
2744#[derive(Clone, Default)]
2745struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2746
2747#[cfg(test)]
2748type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2749
2750#[cfg(test)]
2751impl fmt::Debug for AfterFirstSpawnHook {
2752    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2753        f.write_str("AfterFirstSpawnHook")
2754    }
2755}
2756
2757#[cfg(test)]
2758impl AfterFirstSpawnHook {
2759    fn run(&self, module_id: &str) {
2760        if let Some(hook) = &self.0 {
2761            hook(module_id);
2762        }
2763    }
2764}
2765
2766impl Supervisor {
2767    #[cfg(test)]
2768    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2769        let supervisor = Self::new(registry, policy);
2770        #[cfg(target_os = "macos")]
2771        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2772        supervisor
2773    }
2774    /// Verify the trampoline once when configured. Missing private OS support
2775    /// refuses every macOS launch by name but does not stop the daemon's control
2776    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2777    /// entry point; the library must not exec an arbitrary hosting program.
2778    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2779        let path = path.into();
2780        #[cfg(target_os = "macos")]
2781        {
2782            let result = probe_privacy_trampoline(&path).map(|()| path);
2783            if let Err(cause) = &result {
2784                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2785            }
2786            self.child_roster.set_privacy_trampoline(result);
2787        }
2788        #[cfg(not(target_os = "macos"))]
2789        let _ = path;
2790        self
2791    }
2792    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2793    /// capability-requirement log to read.
2794    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2795        self.child_roster.shutdown_flag()
2796    }
2797
2798    /// The first step of an announced daemon shutdown, before the notice and
2799    /// before any connection is closed.
2800    ///
2801    /// Sets the daemon-shutdown flag first: from here on no module is
2802    /// respawned (crash restart, operator restart, or swap), and every child
2803    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2804    /// the module exits on the EOF this shutdown gives it or is signalled by a
2805    /// service manager that kills the whole cgroup. Then writes the journal's
2806    /// shutdown marker, which records the instant and closes this daemon
2807    /// incarnation's stretch of the journal.
2808    #[cfg(unix)]
2809    pub(crate) fn begin_daemon_shutdown(&self) {
2810        self.child_roster.close();
2811        if let Some(journal) = &self.terminal_journal {
2812            journal.stamp_shutdown();
2813        }
2814    }
2815
2816    /// Announce a cut while established connections can still carry replies.
2817    /// These budgets promise notice and a bounded wait, not child completion;
2818    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2819    #[cfg(unix)]
2820    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2821        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2822        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2823        let Some(forwarding) = &self.forwarding else {
2824            return Ok(());
2825        };
2826        let notice_started = Instant::now();
2827        let modules = forwarding
2828            .begin_daemon_drain()
2829            .map_err(SuperviseError::Forwarding)?;
2830        let deadline_ms =
2831            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2832        let mut notices = tokio::task::JoinSet::new();
2833        let mut drains = Vec::new();
2834        for (module_id, endpoint) in modules {
2835            let Some(target) = forwarding
2836                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2837                .map_err(SuperviseError::Forwarding)?
2838            else {
2839                continue;
2840            };
2841            let routes = forwarding
2842                .endpoint_routes(target.endpoint)
2843                .map_err(SuperviseError::Forwarding)?;
2844            // Restart allows deployed consumers to reopen after the new daemon
2845            // appears. The wire reason stays `restart`; what tells a daemon cut
2846            // apart from a module restart afterwards is the terminal record
2847            // itself, whose disposition is `daemon_shutdown` for every exit
2848            // observed once `begin_daemon_shutdown` has run.
2849            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2850                reason: RouteCloseReason::Restart,
2851                deadline_ms,
2852            })
2853            .expect("module draining serializes");
2854            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2855            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2856            for route in routes {
2857                let client = route.goodbye_target;
2858                if let Some((_, channels)) = clients
2859                    .iter_mut()
2860                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2861                {
2862                    channels.push(client.channel);
2863                } else {
2864                    let channel = client.channel;
2865                    clients.push((client, vec![channel]));
2866                }
2867            }
2868            for (client, mut channels) in clients {
2869                channels.sort_unstable();
2870                channels.dedup();
2871                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2872                    module_id: module_id.clone(),
2873                    channels,
2874                    reason: RouteCloseReason::Restart,
2875                })
2876                .expect("route closing serializes");
2877                recipients.push((client.sink, client.negotiated_ver, closing));
2878            }
2879            for (sink, version, body) in recipients {
2880                notices.spawn(async move {
2881                    let frame = Frame::build_with_version(
2882                        version,
2883                        FrameType::Push,
2884                        control_flags(),
2885                        0,
2886                        0,
2887                        0,
2888                        body,
2889                    )
2890                    .expect("bounded lifecycle notice frame builds");
2891                    sink.send_flushed(frame).await
2892                });
2893            }
2894            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2895            drains.push((module_id, target.endpoint, gauges));
2896        }
2897        // A quiet forwarding table is not proof that queued notices reached the
2898        // socket. Wait for writer flush acknowledgements before testing quiescence.
2899        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2900        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2901            if !matches!(result, Ok(Ok(()))) {
2902                warn!(?result, "daemon shutdown notice delivery failed");
2903            }
2904        }
2905        notices.abort_all();
2906        let notice_elapsed = notice_started.elapsed();
2907        let drain_started = Instant::now();
2908        let deadline = drain_started + DRAIN_BUDGET;
2909        let mut waits = tokio::task::JoinSet::new();
2910        for (module_id, endpoint, gauges) in drains {
2911            let forwarding = Arc::clone(forwarding);
2912            let mut runtime = self.runtime_config();
2913            runtime.health.cadence = Duration::from_millis(100);
2914            waits.spawn(async move {
2915                let wait_result = wait_for_forwarding_quiescence(
2916                    &forwarding,
2917                    &module_id,
2918                    &runtime,
2919                    endpoint,
2920                    deadline,
2921                    &gauges,
2922                    DrainScope::Endpoint(endpoint),
2923                    &mut DrainGaugeTrace::new(&gauges),
2924                )
2925                .await;
2926                // Store the measurement before the task returns: quiescence
2927                // may be reached just as the deadline stops the join loop from
2928                // collecting results. Unfinished or failed waits report false.
2929                forwarding
2930                    .record_daemon_drain_result(
2931                        endpoint,
2932                        drained_after_quiescence_wait(&wait_result),
2933                    )
2934                    .map_err(SuperviseError::Forwarding)?;
2935                wait_result
2936            });
2937        }
2938        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2939            if !matches!(result, Ok(Ok(true))) {
2940                warn!(?result, "daemon shutdown drain did not reach quiescence");
2941            }
2942        }
2943        // Measure only the forwarding drain, not child teardown or the caller's
2944        // signal delivery and process-reaping delays.
2945        info!(
2946            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2947            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2948            "daemon shutdown forwarding drain complete"
2949        );
2950        Ok(())
2951    }
2952
2953    /// The last step of an announced daemon shutdown, after the notice and the
2954    /// drain: send every registered module a module GOODBYE, the same planned
2955    /// stop signal `ck module stop` gives, then close every connection so each
2956    /// subc module sees EOF and starts its own teardown, then end every
2957    /// supervised child that has not exited
2958    /// by its own deadline (its drain budget, capped). Modules lead their own
2959    /// process groups, so a
2960    /// service manager's group kill no longer reaches them; without this a
2961    /// child that does not stop on EOF (every `protocol: "none"` child, which
2962    /// has no connection) would outlive the daemon. Every wait is bounded (see
2963    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2964    #[cfg(unix)]
2965    pub(crate) async fn end_children_for_daemon_shutdown(
2966        &self,
2967        already_escalated: bool,
2968        escalate: impl std::future::Future<Output = ()>,
2969    ) {
2970        tokio::pin!(escalate);
2971        let mut escalated = already_escalated;
2972        if let Some(forwarding) = &self.forwarding {
2973            let reason = CloseReason::new(
2974                "daemon_shutdown",
2975                "the daemon is exiting after its shutdown notice and drain",
2976            );
2977            if escalated {
2978                // The operator asked to stop waiting: queue the GOODBYEs but
2979                // do not wait for them to be written.
2980                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2981            } else {
2982                tokio::select! {
2983                    biased;
2984                    _ = escalate.as_mut() => {
2985                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2986                        escalated = true;
2987                    }
2988                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2989                }
2990            }
2991            let closed = forwarding.close_all_connections(&reason);
2992            debug!(closed, "closed established connections for daemon shutdown");
2993        }
2994        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2995        // already completed and must not be polled again; the child shutdown
2996        // wait is told it is escalated and gets a future that never fires.
2997        let escalated_here = escalated && !already_escalated;
2998        let remaining_escalate = async move {
2999            if escalated_here {
3000                std::future::pending::<()>().await;
3001            } else {
3002                escalate.await;
3003            }
3004        };
3005        crate::child_roster::end_children_for_daemon_shutdown(
3006            &self.child_roster,
3007            escalated,
3008            remaining_escalate,
3009        )
3010        .await;
3011    }
3012
3013    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3014        Self {
3015            registry,
3016            restart_policy,
3017            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3018            connection_file_path: None,
3019            capture_logs_dir: None,
3020            forwarding: None,
3021            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3022            supervisor_handle: None,
3023            health: HealthConfig::default(),
3024            daemon_start_clock: crate::clock::StartClock::capture(),
3025            terminal_journal: None,
3026            spawn_events: SpawnEventFeed::default(),
3027            provenance_probe: ExecutableIdentityProbe::default(),
3028            child_roster: ChildRoster::default(),
3029            #[cfg(target_os = "linux")]
3030            cgroup_placement: None,
3031            #[cfg(test)]
3032            test_after_first_spawn: AfterFirstSpawnHook::default(),
3033        }
3034    }
3035
3036    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3037        self.drain_timeout = drain_timeout;
3038        self
3039    }
3040
3041    /// The drain budget a module gets when its config entry sets none.
3042    pub(crate) fn default_drain_timeout(&self) -> Duration {
3043        self.drain_timeout
3044    }
3045
3046    pub fn with_process_liveness(
3047        mut self,
3048        process_liveness: Arc<SupervisorProcessLiveness>,
3049    ) -> Self {
3050        self.process_liveness = process_liveness;
3051        self
3052    }
3053
3054    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3055        self.connection_file_path = Some(connection_file_path.into());
3056        self
3057    }
3058
3059    /// Enables daemon-owned capture files for supervised stdout and stderr.
3060    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3061        self.capture_logs_dir = Some(logs_dir.into());
3062        self
3063    }
3064
3065    /// Names this daemon lifetime in spawn events, independently of whether a
3066    /// terminal journal is configured.
3067    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3068        // A millisecond start stamp can repeat after clock rollback or a rapid
3069        // restart. Use the connection file's random daemon_id instead: it already
3070        // identifies this daemon lifetime independently of the wall clock.
3071        self.spawn_events.configure_incarnation(daemon_incarnation);
3072        self
3073    }
3074
3075    /// Enables best-effort history shared by every supervised module. Without
3076    /// it, terminal history is kept only in each module's in-memory ring.
3077    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3078        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3079        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3080            path,
3081            daemon_incarnation,
3082        )));
3083        this
3084    }
3085
3086    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3087        self.forwarding = Some(forwarding);
3088        self
3089    }
3090
3091    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3092        self.spawn_events = supervisor_handle.spawn_events.clone();
3093        self.supervisor_handle = Some(supervisor_handle);
3094        self
3095    }
3096
3097    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3098        self.health = health;
3099        self
3100    }
3101
3102    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3103    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3104    /// record is kept.
3105    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3106        self.child_roster.record_to(path.into());
3107        self
3108    }
3109
3110    #[cfg(target_os = "linux")]
3111    pub fn with_cgroup_placement(
3112        mut self,
3113        cgroup_placement: Option<subc_cgroup::Placement>,
3114    ) -> Self {
3115        self.cgroup_placement = cgroup_placement;
3116        self
3117    }
3118
3119    /// Spawn `spec.program` and start monitoring it.
3120    ///
3121    /// The child is expected to parse `--subc <connection-file-path>`, read the
3122    /// TCP+key connection file, authenticate to the already-running listener, and
3123    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3124    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3125        validate_spec(&spec)?;
3126        self.establish_identity(&spec);
3127
3128        let runtime = self.runtime_config();
3129        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3130        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3131        let registration_watch = watch_registration(&self.registry, &spec);
3132        let spawned = spawn_child(
3133            &spec,
3134            runtime.connection_file_path.as_deref(),
3135            self.supervisor_handle.as_ref(),
3136            &runtime.stderr_ring,
3137            runtime.capture_logs_dir.as_deref(),
3138            &runtime.child_roster,
3139            #[cfg(target_os = "linux")]
3140            runtime.cgroup_placement.as_ref(),
3141        );
3142        #[cfg(test)]
3143        self.test_after_first_spawn.run(&spec.module_id);
3144        let mut child = match spawned {
3145            Ok(child) => child,
3146            Err(err) => {
3147                // Unlike the configured paths, a failed `spawn` leaves nothing
3148                // on the roster, so the module must not stay marked configured.
3149                self.abandon_unrostered(&spec.module_id);
3150                return Err(err);
3151            }
3152        };
3153        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3154        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3155        if let Some(spawn) = activity_spawn {
3156            spawn.complete(&snapshot);
3157        }
3158
3159        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3160    }
3161
3162    /// Make `spec`'s module count as configured, with its identity gates
3163    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3164    /// exists.
3165    ///
3166    /// Every path that takes on a new module calls this before `spawn_child`.
3167    /// The order is the point: the child can connect, register, sync its
3168    /// scopes and ask about them as soon as it is spawned, and the module is
3169    /// only put on the roster after `spawn_child` returns. Were the mark set
3170    /// with the roster entry, a fast child would see its own owner reported
3171    /// as not configured, and a scoped `route.open` in that window would be
3172    /// refused as terminal `scope_not_live` ("will never sync") instead of
3173    /// retryable `scope_not_synced`.
3174    fn establish_identity(&self, spec: &ModuleSpec) {
3175        if let Some(supervisor_handle) = &self.supervisor_handle {
3176            supervisor_handle.apply_identity_configuration(spec);
3177            supervisor_handle.mark_configured(&spec.module_id);
3178        }
3179    }
3180
3181    /// Take back [`Self::establish_identity`]'s configured mark when the
3182    /// module will not be put on the roster after all.
3183    fn abandon_unrostered(&self, module_id: &str) {
3184        if let Some(supervisor_handle) = &self.supervisor_handle {
3185            supervisor_handle.unmark_configured_unless_rostered(module_id);
3186        }
3187    }
3188
3189    /// Record a freshly spawned first process as running. On failure the
3190    /// module never reaches the roster, so its configured mark is taken back.
3191    fn mark_first_process_running(
3192        &self,
3193        spec: &ModuleSpec,
3194        runtime: &SupervisorRuntimeConfig,
3195        snapshot: &SharedSnapshot,
3196        child: &mut SupervisedChild,
3197    ) -> Result<(), SuperviseError> {
3198        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3199            self.abandon_unrostered(&spec.module_id);
3200            return Err(err);
3201        }
3202        self.process_liveness
3203            .track(spec.module_id.clone(), Arc::clone(snapshot));
3204        Ok(())
3205    }
3206
3207    /// Start supervising a module declared in daemon configuration.
3208    ///
3209    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3210    /// failures in the supervisor handle so operator-facing `supervisor.list`
3211    /// reflects every configured module while daemon startup continues.
3212    pub fn supervise_configured(
3213        &self,
3214        spec: ModuleSpec,
3215        enabled: bool,
3216    ) -> Result<SupervisedModule, SuperviseError> {
3217        self.supervise_configured_requested(
3218            spec,
3219            enabled,
3220            self.health.clone(),
3221            None,
3222            self.restart_policy,
3223            DEFAULT_REGISTER_TIMEOUT,
3224            None,
3225        )
3226    }
3227
3228    /// Supervise a configured module with its own health, drain, and crash
3229    /// budget. The restart policy is per-module because the config file is:
3230    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3231    /// module that is expensive to restart should not be forced onto the same
3232    /// budget as one that is cheap.
3233    pub fn supervise_configured_with_health(
3234        &self,
3235        spec: ModuleSpec,
3236        enabled: bool,
3237        health: HealthConfig,
3238        drain_timeout_ms: Option<u64>,
3239        restart_policy: RestartPolicy,
3240    ) -> Result<SupervisedModule, SuperviseError> {
3241        self.supervise_configured_requested(
3242            spec,
3243            enabled,
3244            health,
3245            drain_timeout_ms,
3246            restart_policy,
3247            DEFAULT_REGISTER_TIMEOUT,
3248            None,
3249        )
3250    }
3251
3252    #[allow(clippy::too_many_arguments)]
3253    pub(crate) fn supervise_configured_requested(
3254        &self,
3255        spec: ModuleSpec,
3256        enabled: bool,
3257        health: HealthConfig,
3258        drain_timeout_ms: Option<u64>,
3259        restart_policy: RestartPolicy,
3260        register_timeout: Duration,
3261        requester: Option<Principal>,
3262    ) -> Result<SupervisedModule, SuperviseError> {
3263        let registered = self.register_configured_with_health(
3264            spec,
3265            enabled,
3266            health,
3267            drain_timeout_ms,
3268            restart_policy,
3269            register_timeout,
3270        )?;
3271        let daemon_initiated = requester.is_none();
3272        self.start_registered_configured(registered, requester, daemon_initiated)
3273    }
3274
3275    /// Record a configured module as starting or disabled, without launching a
3276    /// process or running its monitor. Bootstrap registers the entire roster
3277    /// before publishing discovery, then activates it once children can read
3278    /// the connection file passed in their `--subc` argument.
3279    pub(crate) fn register_configured_with_health(
3280        &self,
3281        spec: ModuleSpec,
3282        enabled: bool,
3283        health: HealthConfig,
3284        drain_timeout_ms: Option<u64>,
3285        restart_policy: RestartPolicy,
3286        register_timeout: Duration,
3287    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3288        validate_spec(&spec)?;
3289        self.establish_identity(&spec);
3290
3291        let mut runtime = self.runtime_config();
3292        runtime.health = health;
3293        runtime.restart_policy = restart_policy;
3294        runtime.register_timeout = register_timeout;
3295        if let Some(ms) = drain_timeout_ms {
3296            runtime.drain_timeout = Duration::from_millis(ms);
3297            *runtime
3298                .effective_drain_timeout
3299                .lock()
3300                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3301        }
3302        let snapshot = if enabled {
3303            SupervisorSnapshot::starting()
3304        } else {
3305            SupervisorSnapshot::disabled()
3306        };
3307        let snapshot = {
3308            let mut snapshot = snapshot;
3309            if enabled {
3310                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3311                {
3312                    snapshot.waiting_for_previous_pid = Some(pid);
3313                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3314                }
3315            }
3316            snapshot
3317        };
3318        let (module, commands) =
3319            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3320        Ok(RegisteredConfiguredModule {
3321            spec,
3322            runtime,
3323            module,
3324            commands,
3325        })
3326    }
3327
3328    /// Activate a previously registered module. Queued control requests are
3329    /// served by the monitor after the initial configured spawn has settled.
3330    pub(crate) fn start_registered_configured(
3331        &self,
3332        registered: RegisteredConfiguredModule,
3333        requester: Option<Principal>,
3334        daemon_initiated: bool,
3335    ) -> Result<SupervisedModule, SuperviseError> {
3336        let RegisteredConfiguredModule {
3337            spec,
3338            mut runtime,
3339            module,
3340            commands,
3341        } = registered;
3342        runtime.start_requester = requester;
3343        runtime.recovery_is_daemon = daemon_initiated;
3344        let snapshot = &module.inner.snapshot;
3345        let should_spawn = {
3346            let state = lock_snapshot(snapshot)?;
3347            state.enabled && state.waiting_for_previous_pid.is_none()
3348        };
3349        let child = if should_spawn {
3350            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3351            let registration_watch = watch_registration(&self.registry, &spec);
3352            let spawned = spawn_child(
3353                &spec,
3354                runtime.connection_file_path.as_deref(),
3355                self.supervisor_handle.as_ref(),
3356                &runtime.stderr_ring,
3357                runtime.capture_logs_dir.as_deref(),
3358                &runtime.child_roster,
3359                #[cfg(target_os = "linux")]
3360                runtime.cgroup_placement.as_ref(),
3361            );
3362            #[cfg(test)]
3363            self.test_after_first_spawn.run(&spec.module_id);
3364            match spawned {
3365                Ok(mut child) => {
3366                    arm_registration_deadline(
3367                        &mut child,
3368                        runtime.register_timeout,
3369                        registration_watch,
3370                    );
3371                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3372                    if let Some(spawn) = activity_spawn {
3373                        spawn.complete(snapshot);
3374                    }
3375                    Some(child)
3376                }
3377                Err(err) => {
3378                    if runtime.health.critical {
3379                        error!(
3380                            module_id = %spec.module_id,
3381                            program = %spec.program.display(),
3382                            error = %err,
3383                            "critical configured module failed to spawn; marking failed and alerting"
3384                        );
3385                    } else {
3386                        error!(
3387                            module_id = %spec.module_id,
3388                            program = %spec.program.display(),
3389                            error = %err,
3390                            "configured module failed to spawn; marking failed and continuing"
3391                        );
3392                    }
3393                    fail_snapshot(
3394                        snapshot,
3395                        Some(&spec.module_id),
3396                        None,
3397                        &runtime.failure_context(daemon_initiated),
3398                    );
3399                    None
3400                }
3401            }
3402        } else {
3403            None
3404        };
3405        self.start_monitor(spec, runtime, &module, child, commands);
3406        Ok(module)
3407    }
3408
3409    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3410        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3411        SupervisorRuntimeConfig {
3412            registry: self.registry.clone(),
3413            recovery_is_daemon: true,
3414            start_requester: None,
3415            pending_activity_restart: false,
3416            provenance_probe: self.provenance_probe.clone(),
3417            scheduled_respawn: Arc::default(),
3418            deferred_reload_reply: Arc::default(),
3419            restart_policy: self.restart_policy,
3420            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3421            drain_timeout: self.drain_timeout,
3422            // Shared with this module's roster copy: daemon shutdown waits on
3423            // each child for the module's own drain budget, as resolved now.
3424            child_roster: self
3425                .child_roster
3426                .for_module(Arc::clone(&effective_drain_timeout)),
3427            effective_drain_timeout,
3428            default_drain_timeout: self.drain_timeout,
3429            health: self.health.clone(),
3430            connection_file_path: self.connection_file_path.clone(),
3431            capture_logs_dir: self.capture_logs_dir.clone(),
3432            forwarding: self.forwarding.clone(),
3433            supervisor_handle: self.supervisor_handle.clone(),
3434            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3435            terminal_ring: Arc::new(Mutex::new(
3436                TerminalRing::new(
3437                    TerminalRingConfig::default(),
3438                    self.daemon_start_clock.started_at_ms(),
3439                )
3440                .with_start_clock(self.daemon_start_clock)
3441                .with_journal(self.terminal_journal.clone())
3442                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3443            )),
3444            spawn_events: self.spawn_events.clone(),
3445            #[cfg(target_os = "linux")]
3446            cgroup_placement: self.cgroup_placement.clone(),
3447            #[cfg(test)]
3448            test_seed_stale_facts_before_enable_spawn: false,
3449            #[cfg(test)]
3450            test_reload_exit_record_gate: None,
3451        }
3452    }
3453
3454    fn supervised_module(
3455        &self,
3456        spec: ModuleSpec,
3457        runtime: SupervisorRuntimeConfig,
3458        snapshot: SharedSnapshot,
3459        child: Option<SupervisedChild>,
3460    ) -> SupervisedModule {
3461        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3462        self.start_monitor(spec, runtime, &module, child, commands);
3463        module
3464    }
3465
3466    fn register_module(
3467        &self,
3468        spec: &ModuleSpec,
3469        runtime: &SupervisorRuntimeConfig,
3470        snapshot: SharedSnapshot,
3471    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3472        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3473            spec: spec.clone(),
3474            health: runtime.health.clone(),
3475        }));
3476        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3477        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3478        // The module's OWN policy, which may be its per-module config rather than
3479        // the supervisor-wide one; status must report the budget the supervise
3480        // loop actually enforces.
3481        let restart_policy = runtime.restart_policy;
3482        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3483        let (tx, rx) = mpsc::channel(4);
3484        let module_id = spec.module_id.clone();
3485        let module = SupervisedModule {
3486            inner: Arc::new(SupervisedModuleInner {
3487                module_id: module_id.clone(),
3488                registry: Arc::clone(&self.registry),
3489                snapshot,
3490                configuration,
3491                stderr_ring,
3492                terminal_ring,
3493                commands: tx,
3494                monitor: Mutex::new(None),
3495                restart_policy,
3496                effective_drain_timeout,
3497                provenance_probe: self.provenance_probe.clone(),
3498            }),
3499        };
3500        // Identity gates and the configured mark must precede the roster entry,
3501        // so a client never sees a module without its admission policy.
3502        if let Some(supervisor_handle) = &self.supervisor_handle {
3503            supervisor_handle.insert(module.clone());
3504        }
3505        (module, rx)
3506    }
3507
3508    fn start_monitor(
3509        &self,
3510        spec: ModuleSpec,
3511        runtime: SupervisorRuntimeConfig,
3512        module: &SupervisedModule,
3513        child: Option<SupervisedChild>,
3514        commands: mpsc::Receiver<SupervisorCommand>,
3515    ) {
3516        let monitor = tokio::spawn(supervise_loop(
3517            spec,
3518            runtime,
3519            Arc::clone(&self.registry),
3520            Arc::clone(&self.process_liveness),
3521            Arc::clone(&module.inner.snapshot),
3522            child,
3523            commands,
3524        ));
3525        *module
3526            .inner
3527            .monitor
3528            .lock()
3529            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3530    }
3531}
3532
3533impl Default for Supervisor {
3534    fn default() -> Self {
3535        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3536    }
3537}
3538
3539/// Handle to one supervised child process.
3540#[derive(Clone)]
3541pub struct SupervisedModule {
3542    inner: Arc<SupervisedModuleInner>,
3543}
3544
3545struct SupervisedModuleInner {
3546    module_id: String,
3547    registry: Arc<Registry>,
3548    snapshot: SharedSnapshot,
3549    configuration: Arc<Mutex<SupervisedConfiguration>>,
3550    stderr_ring: Arc<Mutex<StderrRing>>,
3551    terminal_ring: Arc<Mutex<TerminalRing>>,
3552    commands: mpsc::Sender<SupervisorCommand>,
3553    monitor: Mutex<Option<JoinHandle<()>>>,
3554    /// Copied from the supervisor's runtime config at spawn so `status()` can
3555    /// report the restart budget without reaching back into the supervisor. The
3556    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3557    restart_policy: RestartPolicy,
3558    effective_drain_timeout: Arc<Mutex<Duration>>,
3559    provenance_probe: ExecutableIdentityProbe,
3560}
3561
3562impl fmt::Debug for SupervisedModule {
3563    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3564        f.debug_struct("SupervisedModule")
3565            .field("module_id", &self.inner.module_id)
3566            .field("status", &self.status())
3567            .finish_non_exhaustive()
3568    }
3569}
3570
3571impl SupervisedModule {
3572    pub fn module_id(&self) -> &str {
3573        &self.inner.module_id
3574    }
3575
3576    /// Test-only: put one probe miss on the streak, the way
3577    /// `handle_health_probe_failure` does, so tests can assert what a later
3578    /// event does to the streak without driving the whole probe loop.
3579    #[cfg(test)]
3580    pub(crate) fn record_health_probe_failure_for_test(
3581        &self,
3582        detail: &str,
3583    ) -> Result<(), SuperviseError> {
3584        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3585            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3586            state.health.detail = Some(detail.to_string());
3587        })
3588    }
3589
3590    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3591        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3592    }
3593
3594    pub(crate) fn registration_end_reason(
3595        &self,
3596    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3597        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3598        Ok(match snapshot.state {
3599            ModuleState::Draining if snapshot.draining_to_replace => {
3600                Some(RegistrationEndReason::SupervisorRestart)
3601            }
3602            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3603            ModuleState::Restarting if snapshot.draining_to_replace => {
3604                Some(RegistrationEndReason::SupervisorRestart)
3605            }
3606            _ => None,
3607        })
3608    }
3609
3610    /// The module's retained stderr, newest lines last.
3611    ///
3612    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3613    /// module, `supervisor.list` renders every module, and putting it in the
3614    /// shared snapshot would make each status read carry a payload almost nobody
3615    /// asked for. Callers that want the text ask for it.
3616    pub fn stderr_tail(
3617        &self,
3618        max_lines: Option<usize>,
3619        max_bytes: Option<usize>,
3620    ) -> StderrTailSnapshot {
3621        self.inner
3622            .stderr_ring
3623            .lock()
3624            .unwrap_or_else(|poisoned| poisoned.into_inner())
3625            .snapshot(max_lines, max_bytes)
3626    }
3627
3628    /// The module's bounded terminal history, oldest retained exit first.
3629    ///
3630    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3631    /// daemon whose in-memory history was necessarily reset.
3632    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3633        self.inner
3634            .terminal_ring
3635            .lock()
3636            .unwrap_or_else(|poisoned| poisoned.into_inner())
3637            .snapshot()
3638    }
3639
3640    /// Retained observations from the current ring and all journal generations.
3641    ///
3642    /// Blocking: this reads the journal files. Async callers use
3643    /// [`Self::read_durable_terminal_history`].
3644    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3645        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3646    }
3647
3648    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3649    /// read (up to every retained generation) never occupies a runtime worker.
3650    /// Fails only if the blocking task could not finish (runtime shutdown or a
3651    /// panic in the read).
3652    pub(crate) async fn read_durable_terminal_history(
3653        &self,
3654    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3655        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3656        let module_id = self.inner.module_id.clone();
3657        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3658            .await
3659    }
3660
3661    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3662        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3663            .map(|(status, _)| status)
3664    }
3665
3666    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3667        let state = lock_snapshot(&self.inner.snapshot)?;
3668        Ok(state
3669            .failed_retry
3670            .pending
3671            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3672            .map(|retry| {
3673                unix_ms_now().saturating_add(
3674                    u64::try_from(
3675                        retry
3676                            .deadline
3677                            .saturating_duration_since(Instant::now())
3678                            .as_millis(),
3679                    )
3680                    .unwrap_or(u64::MAX),
3681                )
3682            }))
3683    }
3684
3685    pub(crate) fn record_deliberate_severance(
3686        &self,
3687        identity: ProcessIdentity,
3688    ) -> Result<bool, SuperviseError> {
3689        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3690        if snapshot.pid != Some(identity.pid)
3691            || snapshot.process_start_time != Some(identity.start_time)
3692        {
3693            return Ok(false);
3694        }
3695        snapshot.deliberate_severance = Some(identity);
3696        Ok(true)
3697    }
3698
3699    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3700    ///
3701    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3702    /// does not produce reader-observability logs.
3703    pub(crate) fn status_for_control(
3704        &self,
3705        caller: &'static str,
3706    ) -> Result<ModuleStatus, SuperviseError> {
3707        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3708            .map(|(status, _)| status)
3709    }
3710
3711    fn status_with_snapshot_lock(
3712        &self,
3713        snapshot: &SharedSnapshot,
3714        caller: Option<&'static str>,
3715    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3716        let mut guard = match caller {
3717            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3718            None => lock_snapshot(snapshot)?,
3719        };
3720        // Read the budget through the pruning path so a reader sees the same
3721        // in-window count the restart decision would use, not a stale total.
3722        let restart_count =
3723            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3724        let snapshot = guard.clone();
3725        drop(guard);
3726        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3727            SuperviseError::StatePoisoned {
3728                module_id: Some(self.inner.module_id.clone()),
3729            }
3730        })?;
3731        let registration_active = self
3732            .inner
3733            .registry
3734            .get_module(&self.inner.module_id)
3735            .map_err(SuperviseError::Registry)?
3736            .is_some();
3737        let protocol = snapshot
3738            .spawned_protocol
3739            .unwrap_or(self.declared_protocol()?);
3740        let running_process =
3741            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3742        // Registration is the difference between the two protocols and the only
3743        // one: a subc module that has not registered cannot serve a request even
3744        // though its process is up, and a `none` module never registers at all,
3745        // so requiring it there would pin `live` to false for the whole life of
3746        // a perfectly healthy process.
3747        let live = match protocol {
3748            ModuleProtocol::Subc => running_process && registration_active,
3749            ModuleProtocol::None => running_process,
3750        };
3751
3752        Ok((
3753            ModuleStatus {
3754                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3755                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3756                module_id: self.inner.module_id.clone(),
3757                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3758                state: snapshot.state,
3759                enabled: snapshot.enabled,
3760                process_alive: snapshot.process_alive,
3761                registration_active,
3762                protocol,
3763                live,
3764                restart_count,
3765                lifetime_restarts: snapshot.lifetime_restarts,
3766                spawn_generation: snapshot.spawn_generation,
3767                max_restarts: self.inner.restart_policy.max_restarts,
3768                restart_window: self.inner.restart_policy.window,
3769                drain_timeout,
3770                restart_backoff: self.inner.restart_policy.backoff,
3771                restart_max_backoff: self.inner.restart_policy.max_backoff,
3772                pid: snapshot.reported_pid(),
3773                spawned_at_ms: snapshot.spawned_at_ms,
3774                spawned_from: snapshot.spawned_from,
3775                launch_spec_digest: snapshot.launch_spec_digest,
3776                process_start_time: snapshot.process_start_time,
3777                last_exit: snapshot.last_exit,
3778                health: snapshot.health,
3779            },
3780            snapshot.spawned_file_identity.clone(),
3781        ))
3782    }
3783
3784    #[cfg(test)]
3785    pub(crate) fn hold_snapshot_for_test(
3786        &self,
3787        acquired: std::sync::mpsc::Sender<()>,
3788        hold: Duration,
3789    ) -> std::thread::JoinHandle<()> {
3790        let snapshot = Arc::clone(&self.inner.snapshot);
3791        std::thread::spawn(move || {
3792            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3793            acquired
3794                .send(())
3795                .expect("test receiver waits for snapshot lock");
3796            std::thread::sleep(hold);
3797        })
3798    }
3799
3800    /// The status and the running-image check for `supervisor.provenance`,
3801    /// taken from one status read. The exec acknowledgement can land between
3802    /// two separate reads, and the reply would then pair "no pid yet" with an
3803    /// image observed after the module started, which describes no single
3804    /// moment.
3805    pub(crate) async fn status_and_running_image_agreement(
3806        &self,
3807    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3808        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3809        let image = self
3810            .inner
3811            .provenance_probe
3812            .observe(
3813                status.pid,
3814                status.spawned_from.as_deref(),
3815                identity,
3816                status.process_start_time,
3817            )
3818            .await;
3819        Ok((status, image))
3820    }
3821
3822    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3823        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3824            Ok(snapshot) => snapshot.clone(),
3825            Err(_) => {
3826                return subc_control::RunningImageAgreement::Unavailable {
3827                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3828                };
3829            }
3830        };
3831        self.inner
3832            .provenance_probe
3833            .observe(
3834                snapshot.reported_pid(),
3835                snapshot.spawned_from.as_deref(),
3836                snapshot.spawned_file_identity.clone(),
3837                snapshot.process_start_time,
3838            )
3839            .await
3840    }
3841
3842    /// Memory and CPU time of the module's current process, read now. Only the
3843    /// process the supervisor spawned is read, not processes it has started.
3844    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3845        #[cfg(windows)]
3846        {
3847            let image = lock_snapshot(&self.inner.snapshot)
3848                .ok()
3849                .and_then(|snapshot| {
3850                    snapshot
3851                        .spawned_file_identity
3852                        .as_ref()
3853                        .and_then(|identity| identity.windows_image.clone())
3854                });
3855            match image {
3856                Some(image) => {
3857                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3858                }
3859                None => subc_control::ChildResourceUsage::Unavailable {
3860                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3861                },
3862            }
3863        }
3864        #[cfg(not(windows))]
3865        {
3866            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3867                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3868                Err(_) => {
3869                    return subc_control::ChildResourceUsage::Unavailable {
3870                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3871                    }
3872                }
3873            };
3874            crate::child_resources::read(pid, start_time)
3875        }
3876    }
3877
3878    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3879        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3880        Ok(match snapshot.state {
3881            ModuleState::Restarting => true,
3882            ModuleState::Failed | ModuleState::Disabled => false,
3883            _ if snapshot.enabled && snapshot.respawn_pending => true,
3884            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3885        })
3886    }
3887
3888    #[cfg(test)]
3889    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3890        self.is_warming_with_snapshot_lock(None)
3891    }
3892
3893    pub(crate) fn is_warming_for_control(
3894        &self,
3895        caller: &'static str,
3896    ) -> Result<bool, SuperviseError> {
3897        self.is_warming_with_snapshot_lock(Some(caller))
3898    }
3899
3900    fn is_warming_with_snapshot_lock(
3901        &self,
3902        caller: Option<&'static str>,
3903    ) -> Result<bool, SuperviseError> {
3904        let snapshot = match caller {
3905            Some(caller) => {
3906                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3907            }
3908            None => lock_snapshot(&self.inner.snapshot)?,
3909        }
3910        .clone();
3911        Ok(matches!(
3912            snapshot.state,
3913            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3914        ))
3915    }
3916
3917    /// Drain the module and stop monitoring it.
3918    pub async fn drain(&self) -> Result<(), SuperviseError> {
3919        self.stop().await
3920    }
3921
3922    pub(crate) async fn retire_requested(
3923        &self,
3924        requester: Option<Principal>,
3925    ) -> Result<(), SuperviseError> {
3926        match self.state()? {
3927            ModuleState::Stopped => return Ok(()),
3928            ModuleState::Failed
3929                if lock_snapshot(&self.inner.snapshot)?
3930                    .failed_retry
3931                    .pending
3932                    .is_none() =>
3933            {
3934                return Ok(())
3935            }
3936            ModuleState::Failed
3937            | ModuleState::Starting
3938            | ModuleState::Running
3939            | ModuleState::Unresponsive
3940            | ModuleState::Restarting
3941            | ModuleState::Draining
3942            | ModuleState::Disabled => {}
3943        }
3944
3945        let (reply_tx, reply_rx) = oneshot::channel();
3946        self.inner
3947            .commands
3948            .send(SupervisorCommand::Retire {
3949                requester,
3950                reply: reply_tx,
3951            })
3952            .await
3953            .map_err(|_| SuperviseError::CommandClosed {
3954                module_id: self.inner.module_id.clone(),
3955            })?;
3956        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3957            module_id: self.inner.module_id.clone(),
3958        })?
3959    }
3960
3961    pub async fn stop(&self) -> Result<(), SuperviseError> {
3962        self.stop_requested(None).await
3963    }
3964
3965    pub(crate) async fn stop_requested(
3966        &self,
3967        requester: Option<Principal>,
3968    ) -> Result<(), SuperviseError> {
3969        match self.state()? {
3970            ModuleState::Stopped => return Ok(()),
3971            ModuleState::Failed
3972                if lock_snapshot(&self.inner.snapshot)?
3973                    .failed_retry
3974                    .pending
3975                    .is_none() =>
3976            {
3977                return Ok(())
3978            }
3979            ModuleState::Failed
3980            | ModuleState::Starting
3981            | ModuleState::Running
3982            | ModuleState::Unresponsive
3983            | ModuleState::Restarting
3984            | ModuleState::Draining
3985            | ModuleState::Disabled => {}
3986        }
3987
3988        let (reply_tx, reply_rx) = oneshot::channel();
3989        self.inner
3990            .commands
3991            .send(SupervisorCommand::Drain {
3992                requester,
3993                reply: reply_tx,
3994            })
3995            .await
3996            .map_err(|_| SuperviseError::CommandClosed {
3997                module_id: self.inner.module_id.clone(),
3998            })?;
3999        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4000            module_id: self.inner.module_id.clone(),
4001        })?
4002    }
4003
4004    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4005        self.restart_requested(drain_timeout_ms, None).await
4006    }
4007
4008    pub(crate) async fn restart_requested(
4009        &self,
4010        drain_timeout_ms: Option<u64>,
4011        requester: Option<Principal>,
4012    ) -> Result<(), SuperviseError> {
4013        let queued_at = Instant::now();
4014        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4015        let (reply_tx, reply_rx) = oneshot::channel();
4016        self.inner
4017            .commands
4018            .send(SupervisorCommand::Restart {
4019                requester,
4020                drain_timeout_ms,
4021                received_at_generation,
4022                queued_at,
4023                reply: reply_tx,
4024            })
4025            .await
4026            .map_err(|_| SuperviseError::CommandClosed {
4027                module_id: self.inner.module_id.clone(),
4028            })?;
4029        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4030            module_id: self.inner.module_id.clone(),
4031        })?
4032    }
4033
4034    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
4035    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4036    /// process then drains in the background of the supervise loop) or has
4037    /// failed, leaving the old process serving.
4038    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4039        self.swap_requested(ready_timeout, None).await
4040    }
4041
4042    pub(crate) async fn swap_requested(
4043        &self,
4044        ready_timeout: Option<Duration>,
4045        requester: Option<Principal>,
4046    ) -> Result<(), SuperviseError> {
4047        let (reply_tx, reply_rx) = oneshot::channel();
4048        self.inner
4049            .commands
4050            .send(SupervisorCommand::Swap {
4051                requester,
4052                ready_timeout,
4053                reply: reply_tx,
4054            })
4055            .await
4056            .map_err(|_| SuperviseError::CommandClosed {
4057                module_id: self.inner.module_id.clone(),
4058            })?;
4059        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4060            module_id: self.inner.module_id.clone(),
4061        })?
4062    }
4063
4064    pub async fn reload(&self) -> Result<(), SuperviseError> {
4065        self.reload_requested(None).await
4066    }
4067
4068    pub(crate) async fn reload_requested(
4069        &self,
4070        requester: Option<Principal>,
4071    ) -> Result<(), SuperviseError> {
4072        let (reply_tx, reply_rx) = oneshot::channel();
4073        self.inner
4074            .commands
4075            .send(SupervisorCommand::Reload {
4076                requester,
4077                reply: reply_tx,
4078            })
4079            .await
4080            .map_err(|_| SuperviseError::CommandClosed {
4081                module_id: self.inner.module_id.clone(),
4082            })?;
4083        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4084            module_id: self.inner.module_id.clone(),
4085        })?
4086    }
4087
4088    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4089        self.set_enabled_requested(enabled, None).await
4090    }
4091
4092    pub(crate) async fn set_enabled_requested(
4093        &self,
4094        enabled: bool,
4095        requester: Option<Principal>,
4096    ) -> Result<bool, SuperviseError> {
4097        let (reply_tx, reply_rx) = oneshot::channel();
4098        self.inner
4099            .commands
4100            .send(SupervisorCommand::SetEnabled {
4101                requester,
4102                enabled,
4103                reply: reply_tx,
4104            })
4105            .await
4106            .map_err(|_| SuperviseError::CommandClosed {
4107                module_id: self.inner.module_id.clone(),
4108            })?;
4109        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4110            module_id: self.inner.module_id.clone(),
4111        })?
4112    }
4113
4114    /// The current process's protocol, or the configured protocol when down.
4115    /// A rescan stores the next launch spec without changing how an existing
4116    /// process registers, serves routes, is probed, or exits.
4117    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4118        let configured = self
4119            .inner
4120            .configuration
4121            .lock()
4122            .map_err(|_| SuperviseError::StatePoisoned {
4123                module_id: Some(self.inner.module_id.clone()),
4124            })?
4125            .spec
4126            .protocol;
4127        let state = lock_snapshot(&self.inner.snapshot)?;
4128        Ok(state.spawned_protocol.unwrap_or(configured))
4129    }
4130
4131    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4132        let configuration =
4133            self.inner
4134                .configuration
4135                .lock()
4136                .map_err(|_| SuperviseError::StatePoisoned {
4137                    module_id: Some(self.inner.module_id.clone()),
4138                })?;
4139        Ok((configuration.spec.clone(), configuration.health.clone()))
4140    }
4141
4142    /// Replace this module's launch spec, keeping its health and drain policy,
4143    /// the way a rescan does for a changed config entry. The running process is
4144    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4145    #[cfg(any(test, feature = "test-support"))]
4146    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4147        let (_, health) = self.configuration()?;
4148        let drain_timeout_ms = u64::try_from(
4149            self.inner
4150                .effective_drain_timeout
4151                .lock()
4152                .unwrap_or_else(|poisoned| poisoned.into_inner())
4153                .as_millis(),
4154        )
4155        .ok();
4156        self.update_configuration(spec, health, drain_timeout_ms)
4157            .await
4158    }
4159
4160    pub(crate) async fn update_configuration(
4161        &self,
4162        spec: ModuleSpec,
4163        health: HealthConfig,
4164        drain_timeout_ms: Option<u64>,
4165    ) -> Result<(), SuperviseError> {
4166        if spec.module_id != self.inner.module_id {
4167            return Err(SuperviseError::InvalidSpec {
4168                reason: "a supervised module's module_id cannot be changed".to_string(),
4169            });
4170        }
4171        validate_spec(&spec)?;
4172        let (reply_tx, reply_rx) = oneshot::channel();
4173        self.inner
4174            .commands
4175            .send(SupervisorCommand::UpdateConfiguration {
4176                spec: spec.clone(),
4177                health: health.clone(),
4178                drain_timeout_ms,
4179                reply: reply_tx,
4180            })
4181            .await
4182            .map_err(|_| SuperviseError::CommandClosed {
4183                module_id: self.inner.module_id.clone(),
4184            })?;
4185        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4186            module_id: self.inner.module_id.clone(),
4187        })?;
4188        let mut configuration =
4189            self.inner
4190                .configuration
4191                .lock()
4192                .map_err(|_| SuperviseError::StatePoisoned {
4193                    module_id: Some(self.inner.module_id.clone()),
4194                })?;
4195        configuration.spec = spec;
4196        configuration.health = health;
4197        Ok(())
4198    }
4199}
4200
4201impl Drop for SupervisedModuleInner {
4202    fn drop(&mut self) {
4203        let Ok(mut monitor) = self.monitor.lock() else {
4204            return;
4205        };
4206        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4207            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4208                state.state = ModuleState::Stopped;
4209                clear_current_process_facts(state);
4210            });
4211            monitor.abort();
4212        }
4213        let _ = monitor.take();
4214    }
4215}
4216
4217#[derive(Debug)]
4218enum SupervisorCommand {
4219    Drain {
4220        requester: Option<Principal>,
4221        reply: oneshot::Sender<Result<(), SuperviseError>>,
4222    },
4223    Retire {
4224        requester: Option<Principal>,
4225        reply: oneshot::Sender<Result<(), SuperviseError>>,
4226    },
4227    Restart {
4228        requester: Option<Principal>,
4229        /// Operator override for this one restart's drain budget, in ms. `None`
4230        /// uses the module's configured/default budget; `Some(0)` cuts
4231        /// immediately (wedge bounce: a stuck request never settles, so
4232        /// waiting only delays recovery).
4233        drain_timeout_ms: Option<u64>,
4234        /// The module's `spawn_generation` when the request was received, before
4235        /// it waited in the command queue. A queued restart whose module has
4236        /// since spawned a newer process is already satisfied (see the handler).
4237        received_at_generation: u64,
4238        /// When the request entered the command queue, so the handler can log
4239        /// how long it waited behind the loop's other work.
4240        queued_at: Instant,
4241        reply: oneshot::Sender<Result<(), SuperviseError>>,
4242    },
4243    Reload {
4244        requester: Option<Principal>,
4245        reply: oneshot::Sender<Result<(), SuperviseError>>,
4246    },
4247    SetEnabled {
4248        requester: Option<Principal>,
4249        enabled: bool,
4250        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4251    },
4252    UpdateConfiguration {
4253        spec: ModuleSpec,
4254        health: HealthConfig,
4255        /// Per-module drain override from the new config; `None` re-resolves to
4256        /// the supervisor-wide default.
4257        drain_timeout_ms: Option<u64>,
4258        reply: oneshot::Sender<()>,
4259    },
4260    Swap {
4261        requester: Option<Principal>,
4262        /// How long the candidate may take to register and declare itself
4263        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4264        ready_timeout: Option<Duration>,
4265        /// Answered at cutover or failure; the incumbent's drain follows.
4266        reply: oneshot::Sender<Result<(), SuperviseError>>,
4267    },
4268}
4269
4270#[derive(Debug)]
4271pub enum SuperviseError {
4272    InvalidSpec {
4273        reason: String,
4274    },
4275    Spawn {
4276        program: PathBuf,
4277        source: io::Error,
4278        cgroup_path: Option<PathBuf>,
4279    },
4280    Cgroup {
4281        module_id: String,
4282        source: io::Error,
4283    },
4284    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4285    /// than spawn a reserved module without its identity binding.
4286    LaunchNonce {
4287        reason: String,
4288    },
4289    Wait {
4290        module_id: String,
4291        source: io::Error,
4292    },
4293    Kill {
4294        module_id: String,
4295        source: io::Error,
4296    },
4297    Forwarding(ForwardingError),
4298    Registry(RegistryError),
4299    ReloadUnavailable {
4300        module_id: String,
4301        reason: String,
4302    },
4303    /// An operator restart/reload was requested for a module that is currently
4304    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4305    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4306    /// by a restart, so these commands are rejected instead of re-enabling it.
4307    Disabled {
4308        module_id: String,
4309    },
4310    ReloadFailed {
4311        module_id: String,
4312        reason: String,
4313    },
4314    RegistrationStillActive {
4315        module_id: String,
4316        waited: Duration,
4317    },
4318    StatePoisoned {
4319        module_id: Option<String>,
4320    },
4321    CommandClosed {
4322        module_id: String,
4323    },
4324    /// A restart or reload arrived while a swap's candidate was warming. The
4325    /// swap owns the module until it cuts over or fails; a stop or disable
4326    /// would have aborted it instead.
4327    SwapInProgress {
4328        module_id: String,
4329    },
4330    /// A swap was refused before anything was spawned.
4331    SwapRefused {
4332        module_id: String,
4333        reason: SwapRefusal,
4334    },
4335    /// A swap spawned a candidate and gave up on it. The candidate has been
4336    /// killed and its slot freed; the incumbent was left serving and was never
4337    /// drained, except in the one `CutoverLost` case described on that arm.
4338    SwapFailed {
4339        module_id: String,
4340        arm: SwapFailureArm,
4341        detail: String,
4342        /// How the candidate exited, when it exited on its own before the
4343        /// supervisor gave up on it.
4344        candidate_exit: Option<ExitReport>,
4345    },
4346}
4347
4348/// Why a swap was refused before a candidate was spawned.
4349#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4350pub enum SwapRefusal {
4351    /// The module's config does not declare `overlap: "safe"`.
4352    OverlapExclusive,
4353    /// The module is not registered, so there is no incumbent to keep serving
4354    /// and nothing a swap would improve on; a plain restart is the tool.
4355    NotRegistered,
4356    /// The module does not speak the subc wire, so a candidate could never
4357    /// register or declare itself ready.
4358    ProtocolNone,
4359    /// The supervisor lacks the forwarding table (to cut routes over) or the
4360    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4361    NotConfigured,
4362    /// A swap is already open for this module.
4363    AlreadySwapping,
4364}
4365
4366impl SwapRefusal {
4367    pub fn as_str(self) -> &'static str {
4368        match self {
4369            Self::OverlapExclusive => "overlap_exclusive",
4370            Self::NotRegistered => "not_registered",
4371            Self::ProtocolNone => "protocol_none",
4372            Self::NotConfigured => "not_configured",
4373            Self::AlreadySwapping => "already_swapping",
4374        }
4375    }
4376}
4377
4378/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4379/// serving and undrained; see `CutoverLost`.
4380#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4381pub enum SwapFailureArm {
4382    /// The candidate process could not be started.
4383    SpawnFailed,
4384    /// The candidate did not register within the readiness budget.
4385    NeverRegistered,
4386    /// The candidate registered but did not declare itself ready in time.
4387    NeverReady,
4388    /// The candidate exited before cutover.
4389    CandidateExited,
4390    /// The candidate declared itself ready but failed its health probe.
4391    CandidateUnhealthy,
4392    /// An operator stop, disable or retire arrived while the candidate warmed.
4393    /// The candidate was killed and the operator's command then carried out on
4394    /// the incumbent.
4395    Interrupted,
4396    /// The candidate's connection closed at the moment of cutover. If it
4397    /// closed before forwarding moved, the incumbent is untouched. If it closed
4398    /// between the forwarding and registry halves of cutover, forwarding can no
4399    /// longer route to the incumbent, so the module is restarted plainly.
4400    CutoverLost,
4401}
4402
4403impl SwapFailureArm {
4404    pub fn as_str(self) -> &'static str {
4405        match self {
4406            Self::SpawnFailed => "spawn_failed",
4407            Self::NeverRegistered => "never_registered",
4408            Self::NeverReady => "never_ready",
4409            Self::CandidateExited => "candidate_exited",
4410            Self::CandidateUnhealthy => "candidate_unhealthy",
4411            Self::Interrupted => "interrupted",
4412            Self::CutoverLost => "cutover_lost",
4413        }
4414    }
4415}
4416
4417impl fmt::Display for SuperviseError {
4418    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4419        match self {
4420            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4421            Self::Spawn {
4422                program,
4423                source,
4424                cgroup_path: Some(cgroup_path),
4425            } => write!(
4426                f,
4427                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4428                cgroup_path.display(),
4429                program.display()
4430            ),
4431            Self::Spawn {
4432                program,
4433                source,
4434                cgroup_path: None,
4435            } => write!(
4436                f,
4437                "failed to spawn module '{}': {source}",
4438                program.display()
4439            ),
4440            Self::Cgroup { module_id, source } => {
4441                write!(
4442                    f,
4443                    "failed to prepare cgroup for module '{module_id}': {source}"
4444                )
4445            }
4446            Self::LaunchNonce { reason } => {
4447                write!(
4448                    f,
4449                    "failed to generate reserved-module launch nonce: {reason}"
4450                )
4451            }
4452            Self::Wait { module_id, source } => {
4453                write!(f, "failed to wait for module '{module_id}': {source}")
4454            }
4455            Self::Kill { module_id, source } => {
4456                write!(f, "failed to kill module '{module_id}': {source}")
4457            }
4458            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4459            Self::Registry(err) => write!(f, "registry error: {err}"),
4460            Self::ReloadUnavailable { module_id, reason } => {
4461                write!(f, "reload unavailable for module '{module_id}': {reason}")
4462            }
4463            Self::Disabled { module_id } => {
4464                write!(
4465                    f,
4466                    "module '{module_id}' is disabled; enable it before restart or reload"
4467                )
4468            }
4469            Self::ReloadFailed { module_id, reason } => {
4470                write!(f, "reload failed for module '{module_id}': {reason}")
4471            }
4472            Self::RegistrationStillActive { module_id, waited } => write!(
4473                f,
4474                "module '{module_id}' registration remained active after waiting {waited:?}"
4475            ),
4476            Self::StatePoisoned { module_id } => match module_id {
4477                Some(module_id) => {
4478                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4479                }
4480                None => write!(f, "supervisor state was poisoned"),
4481            },
4482            Self::CommandClosed { module_id } => {
4483                write!(
4484                    f,
4485                    "supervisor command channel for module '{module_id}' is closed"
4486                )
4487            }
4488            Self::SwapInProgress { module_id } => write!(
4489                f,
4490                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4491            ),
4492            Self::SwapRefused { module_id, reason } => match reason {
4493                SwapRefusal::OverlapExclusive => write!(
4494                    f,
4495                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4496                ),
4497                SwapRefusal::NotRegistered => write!(
4498                    f,
4499                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4500                ),
4501                SwapRefusal::ProtocolNone => write!(
4502                    f,
4503                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4504                ),
4505                SwapRefusal::NotConfigured => write!(
4506                    f,
4507                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4508                ),
4509                SwapRefusal::AlreadySwapping => {
4510                    write!(f, "module '{module_id}' is already being swapped")
4511                }
4512            },
4513            Self::SwapFailed {
4514                module_id,
4515                arm,
4516                detail,
4517                ..
4518            } => write!(
4519                f,
4520                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4521                arm.as_str()
4522            ),
4523        }
4524    }
4525}
4526
4527impl Error for SuperviseError {
4528    fn source(&self) -> Option<&(dyn Error + 'static)> {
4529        match self {
4530            Self::Spawn { source, .. }
4531            | Self::Cgroup { source, .. }
4532            | Self::Wait { source, .. }
4533            | Self::Kill { source, .. } => Some(source),
4534            Self::Forwarding(err) => Some(err),
4535            Self::Registry(err) => Some(err),
4536            Self::LaunchNonce { .. }
4537            | Self::InvalidSpec { .. }
4538            | Self::ReloadUnavailable { .. }
4539            | Self::Disabled { .. }
4540            | Self::ReloadFailed { .. }
4541            | Self::RegistrationStillActive { .. }
4542            | Self::StatePoisoned { .. }
4543            | Self::CommandClosed { .. }
4544            | Self::SwapInProgress { .. }
4545            | Self::SwapRefused { .. }
4546            | Self::SwapFailed { .. } => None,
4547        }
4548    }
4549}
4550
4551pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4552    if spec.module_id.trim().is_empty() {
4553        return Err(SuperviseError::InvalidSpec {
4554            reason: "module_id must not be empty".to_string(),
4555        });
4556    }
4557
4558    Ok(())
4559}
4560
4561#[derive(Debug, Default)]
4562struct HealthProbeRuntime {
4563    spawn_generation: Option<u64>,
4564    configured_health: Option<HealthConfig>,
4565    registered_connection: Option<crate::ConnectionId>,
4566    advertised: bool,
4567    next_probe_at: Option<Instant>,
4568    probe_index: u64,
4569}
4570
4571fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4572    lock_snapshot(snapshot)
4573        .ok()
4574        .and_then(|state| state.spawned_protocol)
4575        .unwrap_or(spec.protocol)
4576}
4577
4578impl HealthProbeRuntime {
4579    fn refresh_registration(
4580        &mut self,
4581        spec: &ModuleSpec,
4582        runtime: &SupervisorRuntimeConfig,
4583        registry: &Registry,
4584        snapshot: &SharedSnapshot,
4585    ) {
4586        let generation = lock_snapshot(snapshot)
4587            .ok()
4588            .map(|state| state.spawn_generation);
4589        if self.spawn_generation != generation {
4590            // A replacement cannot reuse a deadline that is already due for its
4591            // predecessor. HTTP children get a full cadence to start; wire
4592            // children still wait for their own health-advertising HELLO.
4593            *self = Self {
4594                spawn_generation: generation,
4595                ..Self::default()
4596            };
4597        }
4598        if self.configured_health.as_ref() != Some(&runtime.health) {
4599            self.configured_health = Some(runtime.health.clone());
4600            self.next_probe_at = None;
4601            self.registered_connection = None;
4602            self.probe_index = 0;
4603        }
4604        // A non-wire process never registers. Only an explicitly configured
4605        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4606        // health failure for that kind of process.
4607        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4608            self.registered_connection = None;
4609            self.advertised = runtime.health.http.is_some();
4610            if !self.advertised {
4611                self.next_probe_at = None;
4612                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4613                    let unknown = ModuleHealthStatus::default();
4614                    if state.health != unknown {
4615                        state.health = unknown;
4616                    }
4617                });
4618            } else if self.next_probe_at.is_none() {
4619                self.next_probe_at = Some(
4620                    Instant::now()
4621                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4622                );
4623            }
4624            return;
4625        }
4626
4627        let registration = match registry.get_module(&spec.module_id) {
4628            Ok(registration) => registration,
4629            Err(err) => {
4630                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4631                self.advertised = false;
4632                self.next_probe_at = None;
4633                return;
4634            }
4635        };
4636
4637        let Some(registration) = registration else {
4638            self.registered_connection = None;
4639            self.advertised = false;
4640            self.next_probe_at = None;
4641            return;
4642        };
4643
4644        let advertised = registration
4645            .control_ops
4646            .iter()
4647            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4648        if !advertised {
4649            self.registered_connection = Some(registration.connection_id);
4650            self.advertised = false;
4651            self.next_probe_at = None;
4652            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4653                if state.health.status != SupervisorHealthStatus::Unknown
4654                    || state.health.consecutive_failures != 0
4655                    || state.health.last_probe_ms.is_some()
4656                    || state.health.detail.is_some()
4657                    || state.health.metrics.is_some()
4658                {
4659                    state.health.status = SupervisorHealthStatus::Unknown;
4660                    state.health.consecutive_failures = 0;
4661                    state.health.last_probe_ms = None;
4662                    state.health.detail = None;
4663                    state.health.metrics = None;
4664                }
4665            });
4666            return;
4667        }
4668
4669        let reregistered = self.registered_connection != Some(registration.connection_id);
4670        self.registered_connection = Some(registration.connection_id);
4671        self.advertised = true;
4672        if reregistered || self.next_probe_at.is_none() {
4673            self.probe_index = 0;
4674            self.next_probe_at = Some(
4675                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4676            );
4677            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4678                state.health.status = SupervisorHealthStatus::Unknown;
4679                state.health.consecutive_failures = 0;
4680                state.health.detail = None;
4681                state.health.metrics = None;
4682            });
4683        }
4684    }
4685
4686    fn wake_at(&self) -> Option<Instant> {
4687        if !self.advertised {
4688            return None;
4689        }
4690        self.next_probe_at
4691    }
4692
4693    fn due(&self) -> bool {
4694        self.advertised
4695            && self
4696                .next_probe_at
4697                .is_some_and(|next| Instant::now() >= next)
4698    }
4699
4700    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4701        self.probe_index = self.probe_index.wrapping_add(1);
4702        self.next_probe_at = Some(
4703            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4704        );
4705    }
4706}
4707
4708/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4709///
4710/// This was a struct with a single `message: String`, and every one of the
4711/// fifteen construction sites collapsed into it. Each site knows exactly what it
4712/// saw -- the lane is gone, the module did not answer in time, the module
4713/// answered with the wrong thing -- and `handle_health_probe_failure` then
4714/// treated all of them identically: increment a counter, compare to a threshold,
4715/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4716/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4717///
4718/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4719///
4720/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4721///   answer on it again.
4722/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4723///   AND with a perfectly healthy one that lost a CPU race -- which is what
4724///   happens under machine load, and is how this supervisor killed a healthy
4725///   module three times in one day.
4726/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4727///   Restarting on it is defensible, but it is not the silence case and should
4728///   never be counted as one.
4729/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4730///   anything, so it cannot be evidence about the module at all.
4731///
4732/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4733/// one that fires most often, and while every variant collapsed into one string
4734/// it carried the same weight as the strongest.
4735///
4736/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4737/// DESIGN and a reader stopping at it gets the build backwards: the restart
4738/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4739/// probes still increment the failure streak and drive escalation at the
4740/// threshold (see `is_proof_of_death` below for why that is deliberate and
4741/// what gates the change). Absence of evidence restarts modules today.
4742#[derive(Debug)]
4743enum HealthProbeEvidence {
4744    /// The module's control lane is gone. Proof of death.
4745    LaneDead,
4746    /// No reply within the deadline. Proves nothing about the module's state.
4747    NoAnswer,
4748    /// The module replied, but not with a usable health report. Proves it is alive.
4749    BadAnswer,
4750    /// The daemon could not ask. Says nothing about the module.
4751    Misconfigured,
4752}
4753
4754#[derive(Debug)]
4755struct HealthProbeError {
4756    evidence: HealthProbeEvidence,
4757    message: String,
4758}
4759
4760impl HealthProbeError {
4761    fn lane_dead(message: impl Into<String>) -> Self {
4762        Self::with(HealthProbeEvidence::LaneDead, message)
4763    }
4764
4765    fn no_answer(message: impl Into<String>) -> Self {
4766        Self::with(HealthProbeEvidence::NoAnswer, message)
4767    }
4768
4769    fn bad_answer(message: impl Into<String>) -> Self {
4770        Self::with(HealthProbeEvidence::BadAnswer, message)
4771    }
4772
4773    fn misconfigured(message: impl Into<String>) -> Self {
4774        Self::with(HealthProbeEvidence::Misconfigured, message)
4775    }
4776
4777    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4778        Self {
4779            evidence,
4780            message: message.into(),
4781        }
4782    }
4783
4784    /// Whether this observation is proof the module cannot serve.
4785    ///
4786    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4787    /// variant that fires under CPU starvation, and treating it as proof is the
4788    /// defect this enum exists to make impossible to reintroduce silently.
4789    ///
4790    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4791    /// to restart also needs a bound for the case it excludes -- a genuinely
4792    /// wedged module, alive but never answering -- and that bound must come from
4793    /// the distribution of real late-answer latencies, which nothing measures
4794    /// yet. Landing the classification first makes the later change a one-line
4795    /// decision against evidence that already exists, rather than two unproven
4796    /// changes at once.
4797    #[allow(dead_code)]
4798    fn is_proof_of_death(&self) -> bool {
4799        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4800    }
4801
4802    /// Short stable label for logs and the health snapshot.
4803    ///
4804    /// An operator reading `ck health` currently cannot tell "the module is gone"
4805    /// from "the module did not answer in five seconds", because both render as
4806    /// prose in the same field. These labels are what make the two
4807    /// distinguishable at a glance, and they are what a later restart-policy
4808    /// change will be argued from.
4809    fn label(&self) -> &'static str {
4810        match self.evidence {
4811            HealthProbeEvidence::LaneDead => "lane-dead",
4812            HealthProbeEvidence::NoAnswer => "no-answer",
4813            HealthProbeEvidence::BadAnswer => "bad-answer",
4814            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4815        }
4816    }
4817}
4818
4819impl fmt::Display for HealthProbeError {
4820    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4821        f.write_str(&self.message)
4822    }
4823}
4824
4825async fn run_health_probe_cycle(
4826    spec: &ModuleSpec,
4827    runtime: &SupervisorRuntimeConfig,
4828    registry: &Registry,
4829    process_liveness: &SupervisorProcessLiveness,
4830    snapshot: &SharedSnapshot,
4831    child: &mut Option<SupervisedChild>,
4832) {
4833    let now_ms = unix_ms_now();
4834    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4835        .then_some(runtime.health.http.as_deref())
4836        .flatten();
4837    let result = match http {
4838        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4839        None => probe_module_health(&spec.module_id, runtime, None).await,
4840    };
4841    match result {
4842        Ok(report) => {
4843            handle_health_report(
4844                spec,
4845                runtime,
4846                registry,
4847                process_liveness,
4848                snapshot,
4849                child,
4850                report,
4851                now_ms,
4852            )
4853            .await;
4854        }
4855        Err(err) => {
4856            if http.is_some() {
4857                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4858                    state.health.status = SupervisorHealthStatus::Failing;
4859                });
4860            }
4861            handle_health_probe_failure(
4862                spec,
4863                runtime,
4864                registry,
4865                process_liveness,
4866                snapshot,
4867                child,
4868                err,
4869                now_ms,
4870            )
4871            .await;
4872        }
4873    }
4874}
4875
4876pub(crate) struct HttpProbeTarget<'a> {
4877    address: std::net::SocketAddr,
4878    localhost: bool,
4879    authority: &'a str,
4880    path: String,
4881}
4882
4883/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4884/// or TLS. A URL cannot turn a local health check into an outbound connection.
4885pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4886    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4887        return Err("must not contain whitespace, controls, or a fragment".into());
4888    }
4889    let rest = url
4890        .strip_prefix("http://")
4891        .ok_or("must use plain http://")?;
4892    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4893    let (authority, suffix) = rest.split_at(split);
4894    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4895        ("::1", rest)
4896    } else {
4897        let split = authority.find(':').unwrap_or(authority.len());
4898        authority.split_at(split)
4899    };
4900    let ip: std::net::IpAddr = match host {
4901        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4902        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4903        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4904    };
4905    let port = if port.is_empty() {
4906        80
4907    } else {
4908        port.strip_prefix(':')
4909            .and_then(|p| p.parse::<u16>().ok())
4910            .filter(|p| *p > 0)
4911            .ok_or("must have a valid nonzero TCP port")?
4912    };
4913    let path = if suffix.is_empty() {
4914        "/".into()
4915    } else if suffix.starts_with('?') {
4916        format!("/{suffix}")
4917    } else {
4918        suffix.into()
4919    };
4920    Ok(HttpProbeTarget {
4921        address: std::net::SocketAddr::new(ip, port),
4922        localhost: host == "localhost",
4923        authority,
4924        path,
4925    })
4926}
4927
4928async fn probe_http_health(
4929    url: &str,
4930    deadline: Duration,
4931) -> Result<HealthReport, HealthProbeError> {
4932    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4933    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4934    // Keep partial diagnostics outside the timed future so cancellation does
4935    // not discard a status line or body bytes already received.
4936    let mut response_status = String::new();
4937    let mut body = Vec::new();
4938    let probe = async {
4939        // Resolve localhost ourselves so a hosts-file override cannot turn
4940        // this into an outbound request, while IPv6-only local servers work.
4941        let connection = match tokio::net::TcpStream::connect(target.address).await {
4942            Err(_) if target.localhost => {
4943                tokio::net::TcpStream::connect((
4944                    std::net::Ipv6Addr::LOCALHOST,
4945                    target.address.port(),
4946                ))
4947                .await
4948            }
4949            result => result,
4950        };
4951        let mut stream = connection.map_err(|error| {
4952            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4953        })?;
4954        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4955            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4956        let mut reader = BufReader::new(stream);
4957        let mut budget = 16 * 1024;
4958        let status = http_line(&mut reader, &mut budget).await?;
4959        let mut words = status.split_ascii_whitespace();
4960        let version = words.next();
4961        let code = words
4962            .next()
4963            .filter(|word| word.len() == 3)
4964            .and_then(|word| word.parse::<u16>().ok());
4965        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4966            || !code.is_some_and(|code| (100..600).contains(&code))
4967        {
4968            return Err(HealthProbeError::bad_answer(format!(
4969                "invalid HTTP status: {status}"
4970            )));
4971        }
4972        let code = code.expect("validated status code");
4973        response_status = status.clone();
4974        let mut length = None;
4975        let mut chunked = false;
4976        loop {
4977            let line = http_line(&mut reader, &mut budget).await?;
4978            if line.is_empty() {
4979                break;
4980            }
4981            if let Some((name, value)) = line.split_once(':') {
4982                if name.eq_ignore_ascii_case("content-length") {
4983                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4984                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4985                    })?);
4986                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4987                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4988                }
4989            }
4990        }
4991        if chunked {
4992            while body.len() < 200 {
4993                let line = http_line(&mut reader, &mut budget).await?;
4994                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4995                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4996                if size == 0 {
4997                    break;
4998                }
4999                let count = size.min((200 - body.len()) as u64) as usize;
5000                let start = body.len();
5001                (&mut reader)
5002                    .take(count as u64)
5003                    .read_to_end(&mut body)
5004                    .await
5005                    .map_err(|error| {
5006                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5007                    })?;
5008                if body.len() - start != count {
5009                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5010                }
5011                if size > count as u64 || body.len() == 200 {
5012                    break;
5013                }
5014                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5015                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5016                }
5017            }
5018        } else {
5019            reader
5020                .take(length.unwrap_or(200).min(200))
5021                .read_to_end(&mut body)
5022                .await
5023                .map_err(|error| {
5024                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5025                })?;
5026        }
5027        if (200..300).contains(&code) {
5028            Ok(HealthReport::ok())
5029        } else {
5030            Err(HealthProbeError::bad_answer(
5031                "HTTP health endpoint returned non-2xx",
5032            ))
5033        }
5034    };
5035    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5036        Err(HealthProbeError::no_answer(format!(
5037            "HTTP probe timed out after {deadline:?}"
5038        )))
5039    });
5040    if let Err(error) = &mut result {
5041        if !response_status.is_empty() {
5042            error.message = format!(
5043                "{}; {response_status}: {}",
5044                error.message,
5045                String::from_utf8_lossy(&body)
5046            );
5047        }
5048    }
5049    result
5050}
5051
5052async fn http_line(
5053    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5054    remaining: &mut usize,
5055) -> Result<String, HealthProbeError> {
5056    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5057    let mut line = Vec::new();
5058    (&mut *reader)
5059        .take(*remaining as u64)
5060        .read_until(b'\n', &mut line)
5061        .await
5062        .map_err(|error| {
5063            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5064        })?;
5065    *remaining -= line.len();
5066    if !line.ends_with(b"\r\n") {
5067        return Err(HealthProbeError::bad_answer(
5068            "HTTP headers are incomplete or exceed 16 KiB",
5069        ));
5070    }
5071    line.truncate(line.len() - 2);
5072    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5073}
5074
5075async fn probe_module_health(
5076    module_id: &str,
5077    runtime: &SupervisorRuntimeConfig,
5078    drain_deadline: Option<Instant>,
5079) -> Result<HealthReport, HealthProbeError> {
5080    let Some(forwarding) = runtime.forwarding.as_ref() else {
5081        return Err(HealthProbeError::misconfigured(
5082            "supervisor was not configured with a forwarding table",
5083        ));
5084    };
5085    let probe_started_at = Instant::now();
5086    let mut deadline = probe_started_at + runtime.health.deadline;
5087    if let Some(drain_deadline) = drain_deadline {
5088        deadline = deadline.min(drain_deadline);
5089    }
5090    let pending = if drain_deadline.is_some() {
5091        forwarding.begin_drain_health_probe_rpc_for(
5092            module_id,
5093            MODULE_CONTROL_OP_HEALTH_CHECK,
5094            probe_started_at,
5095            deadline,
5096        )
5097    } else {
5098        forwarding.begin_health_probe_rpc_for(
5099            module_id,
5100            MODULE_CONTROL_OP_HEALTH_CHECK,
5101            probe_started_at,
5102            deadline,
5103        )
5104    }
5105    .map_err(|err| {
5106        // The endpoint is not registered, so there is no live control lane to
5107        // ask. That is the module being absent, not slow.
5108        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5109    })?;
5110    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5111}
5112
5113/// [`probe_module_health`] for one endpoint rather than the id's active one.
5114///
5115/// A swap probes two processes that no by-id lookup reaches: its candidate
5116/// before cutover, and its superseded incumbent (for busy gauges) while the
5117/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5118/// bounds the by-id drain probe.
5119async fn probe_endpoint_health(
5120    endpoint: crate::ModuleEndpointId,
5121    runtime: &SupervisorRuntimeConfig,
5122    deadline_cap: Option<Instant>,
5123) -> Result<HealthReport, HealthProbeError> {
5124    let Some(forwarding) = runtime.forwarding.as_ref() else {
5125        return Err(HealthProbeError::misconfigured(
5126            "supervisor was not configured with a forwarding table",
5127        ));
5128    };
5129    let probe_started_at = Instant::now();
5130    let mut deadline = probe_started_at + runtime.health.deadline;
5131    if let Some(cap) = deadline_cap {
5132        deadline = deadline.min(cap);
5133    }
5134    let pending = forwarding
5135        .begin_endpoint_health_probe_rpc_for(
5136            endpoint,
5137            MODULE_CONTROL_OP_HEALTH_CHECK,
5138            probe_started_at,
5139            deadline,
5140        )
5141        .map_err(|err| {
5142            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5143        })?;
5144    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5145}
5146
5147/// Send a begun health probe and classify its answer.
5148async fn await_health_probe(
5149    forwarding: &ForwardingTable,
5150    pending: PendingModuleControlRpc,
5151    deadline: Instant,
5152    probe_budget: Duration,
5153) -> Result<HealthReport, HealthProbeError> {
5154    let PendingModuleControlRpc {
5155        endpoint,
5156        module_sink,
5157        negotiated_ver,
5158        corr,
5159        receiver,
5160    } = pending;
5161    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5162        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5163    })?;
5164    let frame = Frame::build_with_version(
5165        negotiated_ver,
5166        FrameType::Request,
5167        control_flags(),
5168        0,
5169        0,
5170        corr,
5171        body,
5172    )
5173    .map_err(|err| {
5174        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5175    })?;
5176
5177    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5178    // blocks waiting for capacity when the module's egress queue is full, and an
5179    // unbounded await here freezes the whole supervision actor (it stops polling
5180    // Child::wait and supervisor commands), making the module unrecoverable
5181    // in-band. On timeout the probe fails like any transport failure.
5182    match timeout_at(deadline, module_sink.send(frame)).await {
5183        Ok(Ok(())) => {}
5184        Ok(Err(err)) => {
5185            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5186            // A closed sink means the module's egress channel is gone -- the
5187            // receiving half is dropped when its connection tears down. Proof.
5188            return Err(HealthProbeError::lane_dead(format!(
5189                "failed to send health.check: {err}"
5190            )));
5191        }
5192        Err(_elapsed) => {
5193            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5194            // A full egress queue means the module is not draining its socket, which
5195            // is consistent with a wedged module AND with one whose reader is merely
5196            // starved. Silence, not proof.
5197            return Err(HealthProbeError::no_answer(
5198                "health.check send timed out before enqueue (module egress full)",
5199            ));
5200        }
5201    }
5202
5203    match timeout_at(deadline, receiver).await {
5204        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5205        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5206        // and those prove it is alive even though the probe failed.
5207        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5208            response.health_report().ok_or_else(|| {
5209                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5210            })
5211        }
5212        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5213            format!("health.check rejected: {}", body.message),
5214        )),
5215        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5216            Err(HealthProbeError::lane_dead(message))
5217        }
5218        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5219            Err(HealthProbeError::bad_answer(message))
5220        }
5221        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5222            Err(HealthProbeError::bad_answer(format!(
5223                "expected module-control op '{expected}', got '{actual}'"
5224            )))
5225        }
5226        // A reply that crosses the deadline before this waiter observes it is
5227        // still proof of life. The forwarding path records its end-to-end latency
5228        // before delivering this classification.
5229        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5230            "module answered health.check after its daemon deadline",
5231        )),
5232        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5233            "health.check waiter was canceled before the module responded",
5234        )),
5235        Err(_) => {
5236            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5237            Err(HealthProbeError::no_answer(format!(
5238                "module did not answer health.check within {probe_budget:?}"
5239            )))
5240        }
5241    }
5242}
5243
5244#[allow(clippy::too_many_arguments)]
5245async fn handle_health_report(
5246    spec: &ModuleSpec,
5247    runtime: &SupervisorRuntimeConfig,
5248    registry: &Registry,
5249    process_liveness: &SupervisorProcessLiveness,
5250    snapshot: &SharedSnapshot,
5251    child: &mut Option<SupervisedChild>,
5252    report: HealthReport,
5253    now_ms: u64,
5254) {
5255    let status = supervisor_health_status(report.status);
5256    let detail = report.detail.clone();
5257    let metrics = truncate_health_metrics(report.metrics);
5258    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5259        state.health.status = status;
5260        state.health.last_probe_ms = Some(now_ms);
5261        state.health.detail = detail.clone();
5262        state.health.metrics = metrics.clone();
5263        state.health.consecutive_failures = 0;
5264        state.failed_retry.observe_health(
5265            report.status == HealthStatus::Ok,
5266            runtime.restart_policy.window,
5267        );
5268    });
5269
5270    let action = match report.status {
5271        HealthStatus::Ok => return,
5272        HealthStatus::Degraded => runtime.health.on_degraded,
5273        HealthStatus::Failing => runtime.health.on_failing,
5274    };
5275    apply_l3_health_action(
5276        spec,
5277        runtime,
5278        registry,
5279        process_liveness,
5280        snapshot,
5281        child,
5282        status,
5283        detail.as_deref(),
5284        action,
5285        now_ms,
5286    )
5287    .await;
5288}
5289
5290#[allow(clippy::too_many_arguments)]
5291async fn handle_health_probe_failure(
5292    spec: &ModuleSpec,
5293    runtime: &SupervisorRuntimeConfig,
5294    registry: &Registry,
5295    process_liveness: &SupervisorProcessLiveness,
5296    snapshot: &SharedSnapshot,
5297    child: &mut Option<SupervisedChild>,
5298    err: HealthProbeError,
5299    now_ms: u64,
5300) {
5301    let threshold = runtime.health.failure_threshold.max(1);
5302    let mut failures = 0;
5303    // Carry the evidence class into the operator-visible detail. Without it,
5304    // "module did not answer within 5s" and "the control lane is gone" are two
5305    // prose strings in the same field, and the reader has to know the codebase to
5306    // tell which one is proof of anything.
5307    let detail = format!("[{}] {err}", err.label());
5308    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5309        state
5310            .failed_retry
5311            .observe_health(false, runtime.restart_policy.window);
5312        // A failed wire probe invalidates the last report, even before the
5313        // restart threshold. HTTP probes already mark failures as Failing.
5314        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5315            state.health.status = SupervisorHealthStatus::Unknown;
5316        }
5317        state.health.last_probe_ms = Some(now_ms);
5318        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5319        state.health.detail = Some(detail.clone());
5320        state.health.metrics = None;
5321        failures = state.health.consecutive_failures;
5322    });
5323
5324    if failures < threshold {
5325        warn!(
5326            module_id = %spec.module_id,
5327            consecutive_failures = failures,
5328            threshold,
5329            evidence = err.label(),
5330            detail = %detail,
5331            "health.check probe failed"
5332        );
5333        return;
5334    }
5335
5336    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5337        state.state = ModuleState::Unresponsive;
5338        state.health.status = SupervisorHealthStatus::Unresponsive;
5339    });
5340    // The evidence class is logged at the kill site because this is the line an
5341    // operator reads after an unexplained restart. A streak of `no-answer` under
5342    // machine load is the known false-positive shape; a `lane-dead` is not.
5343    if runtime.health.critical {
5344        error!(
5345            module_id = %spec.module_id,
5346            status = "unresponsive",
5347            evidence = err.label(),
5348            detail = %detail,
5349            "critical module health alert"
5350        );
5351    } else {
5352        warn!(
5353            module_id = %spec.module_id,
5354            status = "unresponsive",
5355            evidence = err.label(),
5356            detail = %detail,
5357            "module health threshold breached"
5358        );
5359    }
5360    if let Err(err) = health_restart_child(
5361        spec,
5362        runtime,
5363        registry,
5364        process_liveness,
5365        snapshot,
5366        child,
5367        SupervisorHealthStatus::Unresponsive,
5368        Some(&detail),
5369        now_ms,
5370    )
5371    .await
5372    {
5373        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5374    }
5375}
5376
5377#[allow(clippy::too_many_arguments)]
5378async fn apply_l3_health_action(
5379    spec: &ModuleSpec,
5380    runtime: &SupervisorRuntimeConfig,
5381    registry: &Registry,
5382    process_liveness: &SupervisorProcessLiveness,
5383    snapshot: &SharedSnapshot,
5384    child: &mut Option<SupervisedChild>,
5385    status: SupervisorHealthStatus,
5386    detail: Option<&str>,
5387    action: HealthAction,
5388    now_ms: u64,
5389) {
5390    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5391    match action {
5392        HealthAction::Report => {
5393            info!(
5394                module_id = %spec.module_id,
5395                status = ?status,
5396                detail,
5397                "module reported non-ok health"
5398            );
5399        }
5400        HealthAction::Alert => {
5401            error!(
5402                module_id = %spec.module_id,
5403                status = ?status,
5404                detail,
5405                "module health alert"
5406            );
5407        }
5408        HealthAction::Restart => {
5409            if let Err(err) = health_restart_child(
5410                spec,
5411                runtime,
5412                registry,
5413                process_liveness,
5414                snapshot,
5415                child,
5416                status,
5417                detail,
5418                now_ms,
5419            )
5420            .await
5421            {
5422                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5423            }
5424        }
5425    }
5426}
5427
5428#[allow(clippy::too_many_arguments)]
5429async fn health_restart_child(
5430    spec: &ModuleSpec,
5431    runtime: &SupervisorRuntimeConfig,
5432    registry: &Registry,
5433    process_liveness: &SupervisorProcessLiveness,
5434    snapshot: &SharedSnapshot,
5435    child: &mut Option<SupervisedChild>,
5436    status: SupervisorHealthStatus,
5437    detail: Option<&str>,
5438    now_ms: u64,
5439) -> Result<(), SuperviseError> {
5440    let (enabled, schedule) = {
5441        let mut state = lock_snapshot(snapshot)?;
5442        let enabled = state.enabled;
5443        let schedule = if enabled {
5444            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5445        } else {
5446            None
5447        };
5448        (enabled, schedule)
5449    };
5450
5451    if !enabled {
5452        return Err(SuperviseError::Disabled {
5453            module_id: spec.module_id.clone(),
5454        });
5455    }
5456
5457    if schedule.is_none() {
5458        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5459        error!(
5460            module_id = %spec.module_id,
5461            status = ?status,
5462            detail,
5463            max_restarts = runtime.restart_policy.max_restarts,
5464            window_secs = runtime.restart_policy.window.as_secs(),
5465            reason = %runtime.restart_policy.budget_exhausted_detail(),
5466            "health restart budget exhausted; marking module failed"
5467        );
5468        let stop_notice = begin_forwarding_drain_if_configured(
5469            spec,
5470            runtime,
5471            registry,
5472            snapshot,
5473            Some(true),
5474            RouteCloseReason::Disable,
5475        )
5476        .await?;
5477        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5478            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5479        })?;
5480        let result = drain_optional_child(
5481            &spec.module_id,
5482            spec.protocol,
5483            stop_notice,
5484            registry,
5485            runtime.forwarding.as_deref(),
5486            snapshot,
5487            &runtime.terminal_ring,
5488            &runtime.spawn_events,
5489            child,
5490            runtime.drain_timeout,
5491            ModuleState::Failed,
5492            Some(true),
5493            &runtime.failure_context(true),
5494        )
5495        .await;
5496        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5497            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5498                state.failed_retry.schedule(&spec.module_id);
5499            }
5500        })?;
5501        result?;
5502        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5503        return Ok(());
5504    }
5505
5506    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5507    let mut restart_count = 0;
5508    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5509        restart_count = state.crash_restarts.len();
5510        state.state = ModuleState::Unresponsive;
5511        state.health.status = status;
5512        state.health.last_action = Some(HealthAction::Restart.to_string());
5513        state.health.last_action_ms = Some(now_ms);
5514    })?;
5515    warn!(
5516        module_id = %spec.module_id,
5517        status = ?status,
5518        detail,
5519        restart_count,
5520        restart_in_window = schedule.restart_in_window,
5521        delay_ms = schedule.delay.as_millis() as u64,
5522        "health-triggered module restart"
5523    );
5524
5525    let stop_notice = begin_forwarding_drain_if_configured(
5526        spec,
5527        runtime,
5528        registry,
5529        snapshot,
5530        Some(true),
5531        RouteCloseReason::Restart,
5532    )
5533    .await?;
5534    drain_optional_child(
5535        &spec.module_id,
5536        spec.protocol,
5537        stop_notice,
5538        registry,
5539        runtime.forwarding.as_deref(),
5540        snapshot,
5541        &runtime.terminal_ring,
5542        &runtime.spawn_events,
5543        child,
5544        runtime.drain_timeout,
5545        ModuleState::Restarting,
5546        Some(true),
5547        &runtime.failure_context(true),
5548    )
5549    .await?;
5550    schedule_respawn(
5551        runtime,
5552        snapshot,
5553        &spec.module_id,
5554        schedule.delay,
5555        RespawnKind::Spawn,
5556        true,
5557    )
5558}
5559
5560fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5561    if let Some(reply) = runtime
5562        .deferred_reload_reply
5563        .lock()
5564        .unwrap_or_else(|p| p.into_inner())
5565        .take()
5566    {
5567        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5568            module_id: module_id.to_string(),
5569            reason: reason.to_string(),
5570        }));
5571    }
5572}
5573
5574fn schedule_respawn(
5575    runtime: &SupervisorRuntimeConfig,
5576    snapshot: &SharedSnapshot,
5577    module_id: &str,
5578    delay: Duration,
5579    kind: RespawnKind,
5580    daemon_initiated: bool,
5581) -> Result<(), SuperviseError> {
5582    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5583    update_snapshot(snapshot, Some(module_id), |state| {
5584        state.respawn_pending = true
5585    })?;
5586    *runtime
5587        .scheduled_respawn
5588        .lock()
5589        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5590        deadline: Instant::now() + delay,
5591        kind,
5592        daemon_initiated,
5593    });
5594    Ok(())
5595}
5596
5597fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5598    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5599        state.health.last_action = Some(action);
5600        state.health.last_action_ms = Some(now_ms);
5601    });
5602}
5603
5604fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5605    match status {
5606        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5607        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5608        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5609    }
5610}
5611
5612/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5613/// returned to every `supervisor.list` and `supervisor.health` caller.
5614///
5615/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5616/// path: that request exists to return a module's complete metrics object, and
5617/// `ck health <module-id>` documents it as the way to see what the cached view
5618/// truncates. The asymmetry is the feature.
5619///
5620/// So a new caller must decide which side it is on rather than assume the cap is
5621/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5622/// the truncation that path exists to avoid.
5623fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5624    let metrics = metrics?;
5625    match serde_json::to_vec(&metrics) {
5626        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5627            "truncated": true,
5628            "original_bytes": encoded.len(),
5629        })),
5630        Ok(_) | Err(_) => Some(metrics),
5631    }
5632}
5633
5634/// Spread health probes so a fleet-wide restart does not converge them.
5635///
5636/// The delay is derived from the module id and probe index rather than a random
5637/// source, so it is deterministic per module: a module keeps its own offset
5638/// across daemon restarts instead of re-rolling into a collision.
5639fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5640    if cadence.is_zero() {
5641        return Duration::ZERO;
5642    }
5643    let cadence_ms = cadence.as_millis() as u64;
5644    // This early return is REDUNDANT, deliberately, and a mutation run will show
5645    // it surviving removal. Recording why here so the next person to notice does
5646    // not have to re-derive it:
5647    //
5648    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5649    //   a zero cadence and builds the Duration from whole milliseconds, so a
5650    //   sub-millisecond cadence cannot come from config.
5651    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5652    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5653    //   -- exactly what this returns.
5654    //
5655    // Kept as a guard against a future widening of the config parser (accepting
5656    // microseconds, say), which would make the sub-millisecond case reachable.
5657    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5658    // divides by zero. Remove this and nothing changes.
5659    if cadence_ms == 0 {
5660        return cadence;
5661    }
5662    // Note that this never returns less than one cadence, including for the FIRST
5663    // probe. So a freshly registered module reports health `unknown` for a full
5664    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5665    // ready to answer.
5666    //
5667    // That is a property of the supervisor's schedule, not of any module: an
5668    // operator watching a restart sees `unknown` and cannot tell it from a module
5669    // that is slow to warm. Measured on two unrelated modules, both flipping to
5670    // `ok` between 22s and 32s after restart.
5671    //
5672    // Left as-is because spreading the first probe is what keeps a fleet-wide
5673    // restart from firing fourteen simultaneous probes into a cold machine. The
5674    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5675    // that thundering herd for a faster first reading.
5676    let jitter_span = (cadence_ms / 10).max(1);
5677    let hash = module_id.as_bytes().iter().fold(
5678        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5679        |acc, byte| {
5680            acc.wrapping_mul(1099511628211)
5681                .wrapping_add(u64::from(*byte))
5682        },
5683    );
5684    cadence + Duration::from_millis(hash % jitter_span)
5685}
5686
5687#[cfg(test)]
5688mod tests {
5689    use super::*;
5690
5691    #[test]
5692    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5693        let handle = SupervisorHandle::new();
5694        let module_id = "readded-tombstone";
5695        handle.record_rescan_removal(module_id);
5696        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5697
5698        handle.apply_identity_configuration(&ModuleSpec {
5699            module_id: module_id.to_string(),
5700            program: PathBuf::from("/test/module"),
5701            args: Vec::new(),
5702            env: Vec::new(),
5703            reserved: false,
5704            reserved_prefixes: Vec::new(),
5705            protocol: ModuleProtocol::Subc,
5706            overlap: Default::default(),
5707        });
5708
5709        assert!(
5710            handle.removal_tombstone_age_ms(module_id).is_none(),
5711            "a re-added module must not retain a stale removal tombstone"
5712        );
5713    }
5714
5715    /// What one module's owner looked like from the control plane at the
5716    /// instant after its first process was spawned.
5717    #[derive(Debug, PartialEq, Eq)]
5718    struct OwnerInSpawnWindow {
5719        module_id: String,
5720        configured: bool,
5721        on_roster: bool,
5722        admission_refusal: Option<&'static str>,
5723    }
5724
5725    /// A supervised module's process can connect, register, sync its scopes
5726    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5727    /// the module on the roster until afterward; configured supervision does
5728    /// so before spawning. In both cases the owner must
5729    /// already read as configured, so a scoped `route.open` against it is
5730    /// refused as retryable `scope_not_synced` and not as terminal
5731    /// `scope_not_live` ("will never sync").
5732    ///
5733    /// The hook runs just after each initial spawn attempt, before returning to
5734    /// the caller, so no race with a real child's registration is needed.
5735    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5736    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5737        use crate::scopes::ScopeTable;
5738        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5739
5740        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5741        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5742            module_id: module_id.to_string(),
5743            program,
5744            args: Vec::new(),
5745            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5746                .into_iter()
5747                .map(|key| (key.to_string(), dir.path().display().to_string()))
5748                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5749                .collect(),
5750            reserved: false,
5751            reserved_prefixes: Vec::new(),
5752            protocol: ModuleProtocol::Subc,
5753            overlap: Default::default(),
5754        };
5755        let live = super::terminal_history_tests::fake_aft_stub_path();
5756        let missing = dir.path().join("definitely-missing-module");
5757
5758        let handle = SupervisorHandle::new();
5759        let mut supervisor =
5760            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5761                .with_handle(handle.clone());
5762        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5763        let hook_handle = handle.clone();
5764        let hook_observed = Arc::clone(&observed);
5765        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5766            // Exactly what the control plane computes for a scoped route.open
5767            // naming this module as the owner of a scope it has not synced.
5768            let configured = hook_handle.is_configured(module_id);
5769            let selector = ScopeSelector::new(
5770                Principal::Reserved {
5771                    module_id: module_id.to_string(),
5772                },
5773                "s".to_string(),
5774                Some(1),
5775            );
5776            let carrier = Principal::Reserved {
5777                module_id: "carrier".to_string(),
5778            };
5779            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5780                .admit(&carrier, module_id, &selector, configured)
5781            {
5782                Ok(_) => None,
5783                Err(refusal) => Some(refusal.code),
5784            };
5785            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5786                module_id: module_id.to_string(),
5787                configured,
5788                on_roster: hook_handle.get(module_id).is_some(),
5789                admission_refusal,
5790            });
5791        })));
5792
5793        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5794        let configured = supervisor
5795            .supervise_configured(stub("configured", live.clone()), true)
5796            .unwrap();
5797        let with_health = supervisor
5798            .supervise_configured_with_health(
5799                stub("with-health", live.clone()),
5800                true,
5801                HealthConfig::default(),
5802                None,
5803                RestartPolicy::default(),
5804            )
5805            .unwrap();
5806        // The failed-spawn path still puts the module on the roster (as
5807        // failed), so it is configured throughout.
5808        let failed = supervisor
5809            .supervise_configured_with_health(
5810                stub("failed-spawn", missing.clone()),
5811                true,
5812                HealthConfig::default(),
5813                None,
5814                RestartPolicy::default(),
5815            )
5816            .unwrap();
5817        // A failed plain `spawn` puts nothing on the roster, so its mark is
5818        // taken back once the spawn has failed.
5819        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5820
5821        let expected = [
5822            "plain",
5823            "configured",
5824            "with-health",
5825            "failed-spawn",
5826            "spawn-error",
5827        ]
5828        .into_iter()
5829        .map(|module_id| OwnerInSpawnWindow {
5830            module_id: module_id.to_string(),
5831            configured: true,
5832            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5833            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5834        })
5835        .collect::<Vec<_>>();
5836        assert_eq!(*observed.lock().unwrap(), expected);
5837
5838        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5839            assert!(
5840                handle.get(module_id).is_some(),
5841                "{module_id} is on the roster"
5842            );
5843            assert!(
5844                handle.is_configured(module_id),
5845                "{module_id} stays configured"
5846            );
5847        }
5848        assert!(handle.get("spawn-error").is_none());
5849        assert!(
5850            !handle.is_configured("spawn-error"),
5851            "a plain spawn that failed must not leave its module marked configured"
5852        );
5853
5854        // Leaving the roster clears the mark with it.
5855        handle.retire("failed-spawn");
5856        assert!(!handle.is_configured("failed-spawn"));
5857
5858        for module in [plain, configured, with_health] {
5859            module.stop().await.unwrap();
5860        }
5861        drop(failed);
5862    }
5863
5864    /// A configured module reads as starting until the supervisor has admitted
5865    /// its first process (exec confirmed, identity durably recorded), even
5866    /// though that process is already running by then and can register over
5867    /// HELLO, which is what lists it in catalog.list. So a client that needs
5868    /// the supervisor state must wait for that state; seeing the module in
5869    /// the catalog does not mean supervisor.list already says running.
5870    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5871    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5872        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5873        let spec = ModuleSpec {
5874            module_id: "spawned-not-admitted".to_string(),
5875            program: super::terminal_history_tests::fake_aft_stub_path(),
5876            args: Vec::new(),
5877            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5878                .into_iter()
5879                .map(|key| (key.to_string(), dir.path().display().to_string()))
5880                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5881                .collect(),
5882            reserved: false,
5883            reserved_prefixes: Vec::new(),
5884            protocol: ModuleProtocol::Subc,
5885            overlap: Default::default(),
5886        };
5887        let handle = SupervisorHandle::new();
5888        let mut supervisor =
5889            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5890                .with_handle(handle.clone());
5891        let observed = Arc::new(Mutex::new(None));
5892        let hook_handle = handle.clone();
5893        let hook_observed = Arc::clone(&observed);
5894        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5895            let state = hook_handle
5896                .get(module_id)
5897                .expect("a configured module is on the roster before it is spawned")
5898                .state()
5899                .unwrap();
5900            *hook_observed.lock().unwrap() = Some(state);
5901        })));
5902
5903        let module = supervisor.supervise_configured(spec, true).unwrap();
5904        assert_eq!(
5905            *observed.lock().unwrap(),
5906            Some(ModuleState::Starting),
5907            "between the spawn and its admission the supervisor still reports starting"
5908        );
5909        assert_eq!(module.state().unwrap(), ModuleState::Running);
5910        module.stop().await.unwrap();
5911    }
5912
5913    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5914        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5915        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5916            snapshot.process_alive = true;
5917            snapshot.pid = Some(41);
5918            snapshot.spawned_at_ms = Some(42);
5919            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5920            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5921            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5922                device: 43,
5923                inode: 44,
5924                #[cfg(windows)]
5925                windows_image: None,
5926            });
5927        })
5928        .unwrap();
5929        snapshot
5930    }
5931
5932    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5933        let snapshot = lock_snapshot(snapshot).unwrap();
5934        assert!(!snapshot.process_alive);
5935        assert_eq!(snapshot.pid, None);
5936        assert_eq!(snapshot.spawned_at_ms, None);
5937        assert_eq!(snapshot.spawned_from, None);
5938        assert_eq!(snapshot.launch_spec_digest, None);
5939        assert_eq!(snapshot.spawned_file_identity, None);
5940    }
5941
5942    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5943    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5944        let supervisor =
5945            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5946        let mut runtime = supervisor.runtime_config();
5947        runtime.test_seed_stale_facts_before_enable_spawn = true;
5948        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5949        let mut child = None;
5950        let spec = ModuleSpec {
5951            module_id: "failed-enable-clears-facts".to_string(),
5952            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5953            args: Vec::new(),
5954            env: Vec::new(),
5955            reserved: false,
5956            reserved_prefixes: Vec::new(),
5957            protocol: ModuleProtocol::Subc,
5958            overlap: Default::default(),
5959        };
5960
5961        let result = set_child_enabled(
5962            &spec,
5963            &runtime,
5964            &supervisor.registry,
5965            &supervisor.process_liveness,
5966            &snapshot,
5967            &mut child,
5968            true,
5969            &runtime.failure_context(false),
5970        )
5971        .await;
5972
5973        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5974        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5975        assert_snapshot_process_facts_cleared(&snapshot);
5976    }
5977
5978    #[tokio::test]
5979    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5980        let supervisor =
5981            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5982        let runtime = supervisor.runtime_config();
5983        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5984            ModuleState::Restarting,
5985            true,
5986        )));
5987        let spec = ModuleSpec {
5988            module_id: "start-stranded-restarting".to_string(),
5989            program: super::terminal_history_tests::fake_aft_stub_path(),
5990            args: Vec::new(),
5991            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5992            reserved: false,
5993            reserved_prefixes: Vec::new(),
5994            protocol: ModuleProtocol::None,
5995            overlap: Default::default(),
5996        };
5997        let mut child = None;
5998        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5999        assert!(!super::set_child_enabled(
6000            &spec,
6001            &runtime,
6002            &Registry::default(),
6003            &supervisor.process_liveness,
6004            &snapshot,
6005            &mut child,
6006            true,
6007            &runtime.failure_context(false),
6008        )
6009        .await
6010        .unwrap());
6011        assert!(child.is_none());
6012        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6013        assert!(super::set_child_enabled(
6014            &spec,
6015            &runtime,
6016            &Registry::default(),
6017            &supervisor.process_liveness,
6018            &snapshot,
6019            &mut child,
6020            true,
6021            &runtime.failure_context(false),
6022        )
6023        .await
6024        .unwrap());
6025        assert_eq!(
6026            lock_snapshot(&snapshot).unwrap().state,
6027            ModuleState::Running
6028        );
6029        let mut child = child.unwrap();
6030        child.start_kill().unwrap();
6031        child.wait().await.unwrap();
6032    }
6033
6034    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6035    async fn failed_reload_spawn_clears_current_process_facts() {
6036        let supervisor =
6037            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6038        let mut runtime = supervisor.runtime_config();
6039        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6040        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6041        let mut child = None;
6042        let spec = ModuleSpec {
6043            module_id: "failed-reload-clears-facts".to_string(),
6044            program: PathBuf::from("/unused/failed-reload-module"),
6045            args: Vec::new(),
6046            env: Vec::new(),
6047            reserved: false,
6048            reserved_prefixes: Vec::new(),
6049            protocol: ModuleProtocol::Subc,
6050            overlap: Default::default(),
6051        };
6052
6053        let result = handle_reload_spawn_failure(
6054            &spec,
6055            &runtime,
6056            &supervisor.process_liveness,
6057            &snapshot,
6058            &mut child,
6059            "forced reload spawn failure".to_string(),
6060        )
6061        .await;
6062
6063        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6064        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6065        assert_snapshot_process_facts_cleared(&snapshot);
6066    }
6067
6068    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6069    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6070        let supervisor =
6071            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6072        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6073        let module = supervisor.supervised_module(
6074            ModuleSpec {
6075                module_id: "drop-clears-facts".to_string(),
6076                program: PathBuf::from("/unused/drop-module"),
6077                args: Vec::new(),
6078                env: Vec::new(),
6079                reserved: false,
6080                reserved_prefixes: Vec::new(),
6081                protocol: ModuleProtocol::Subc,
6082                overlap: Default::default(),
6083            },
6084            supervisor.runtime_config(),
6085            Arc::clone(&snapshot),
6086            None,
6087        );
6088        assert!(!module
6089            .inner
6090            .monitor
6091            .lock()
6092            .unwrap()
6093            .as_ref()
6094            .unwrap()
6095            .is_finished());
6096
6097        drop(module);
6098
6099        assert_eq!(
6100            lock_snapshot(&snapshot).unwrap().state,
6101            ModuleState::Stopped
6102        );
6103        assert_snapshot_process_facts_cleared(&snapshot);
6104    }
6105
6106    #[cfg(unix)]
6107    #[tokio::test]
6108    async fn rescan_preserves_running_protocol_until_respawn() {
6109        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6110        let initial = ModuleSpec {
6111            module_id: "rescan-protocol".into(),
6112            program: PathBuf::from("/bin/sleep"),
6113            args: vec!["60".into()],
6114            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6115                .into_iter()
6116                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6117                .collect(),
6118            reserved: false,
6119            reserved_prefixes: vec![],
6120            protocol: ModuleProtocol::None,
6121            overlap: Default::default(),
6122        };
6123        let supervisor =
6124            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6125        let module = supervisor.spawn(initial.clone()).unwrap();
6126        assert!(module.status().unwrap().live);
6127        let mut next = initial;
6128        next.protocol = ModuleProtocol::Subc;
6129        module
6130            .update_configuration(next.clone(), HealthConfig::default(), None)
6131            .await
6132            .unwrap();
6133        assert!(
6134            module.status().unwrap().live,
6135            "rescan must not require HELLO from the old non-wire process"
6136        );
6137        let runtime = supervisor.runtime_config();
6138        let action = on_child_exit(
6139            &next,
6140            RestartPolicy::default(),
6141            &supervisor.registry,
6142            &module.inner.snapshot,
6143            &runtime.terminal_ring,
6144            &runtime.spawn_events,
6145            &runtime.child_roster,
6146            ExitReport {
6147                kind: ExitKind::Clean,
6148                code: Some(0),
6149                signal: None,
6150                at_ms: unix_ms_now(),
6151            },
6152            true,
6153        )
6154        .await;
6155        assert!(
6156            matches!(action, NextAction::Restart { .. }),
6157            "the old non-wire process's clean exit must restart"
6158        );
6159        module.drain().await.unwrap();
6160    }
6161
6162    #[cfg(unix)]
6163    #[tokio::test]
6164    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6165        use std::os::unix::fs::PermissionsExt;
6166        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6167        let script = dir.join("module.sh");
6168        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6169        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6170        let record_path = dir.join("live-children.json");
6171        let supervisor =
6172            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6173                .with_live_children_record(&record_path);
6174        for (program, args) in [
6175            (PathBuf::from("sleep"), vec!["60".into()]),
6176            (script, vec![]),
6177        ] {
6178            let spec = ModuleSpec {
6179                module_id: "image-identity".into(),
6180                program: program.clone(),
6181                args,
6182                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6183                    .into_iter()
6184                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6185                    .collect(),
6186                reserved: false,
6187                reserved_prefixes: vec![],
6188                protocol: ModuleProtocol::None,
6189                overlap: Default::default(),
6190            };
6191            let module = supervisor.spawn(spec).unwrap();
6192            #[cfg(target_os = "macos")]
6193            {
6194                // SETEXEC confirmation is asynchronous; the orphan record must
6195                // identify the final image, never the intermediate trampoline.
6196                let deadline = Instant::now() + Duration::from_secs(5);
6197                while crate::live_children::read_record(&record_path)
6198                    .unwrap()
6199                    .iter()
6200                    .all(|entry| entry.executable.is_none())
6201                {
6202                    assert!(Instant::now() < deadline, "module image was not confirmed");
6203                    tokio::time::sleep(Duration::from_millis(5)).await;
6204                }
6205            }
6206            let entry = crate::live_children::read_record(&record_path)
6207                .unwrap()
6208                .pop()
6209                .unwrap();
6210            let observed = subc_os::Process::open(entry.pid)
6211                .unwrap()
6212                .unwrap()
6213                .observe()
6214                .unwrap();
6215            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6216            module.drain().await.unwrap();
6217            assert_eq!(
6218                verdict,
6219                crate::live_children::IdentityVerdict::Matches,
6220                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6221            );
6222        }
6223    }
6224
6225    #[cfg(unix)]
6226    fn http_fixture(
6227        dir: &std::path::Path,
6228        url: &str,
6229        threshold: u32,
6230    ) -> crate::daemon_config::ConfiguredModule {
6231        let path = dir.join("subc.jsonc");
6232        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6233            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6234            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6235            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6236            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6237        }}}).to_string()).unwrap();
6238        crate::daemon_config::load(&path)
6239            .unwrap()
6240            .unwrap()
6241            .modules
6242            .pop()
6243            .unwrap()
6244    }
6245
6246    #[cfg(unix)]
6247    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6248        timeout(Duration::from_secs(5), async {
6249            loop {
6250                if module.status().unwrap().health.status == status {
6251                    break;
6252                }
6253                sleep(Duration::from_millis(5)).await;
6254            }
6255        })
6256        .await
6257        .unwrap_or_else(|_| {
6258            panic!(
6259                "expected {status:?}, got {:?}",
6260                module.status().unwrap().health
6261            )
6262        });
6263    }
6264
6265    #[cfg(unix)]
6266    #[tokio::test]
6267    async fn http_health_status_flips_ok_failing_ok() {
6268        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6269        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6270        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6271        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6272        let serving_status = status.clone();
6273        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6274        let server = tokio::spawn(async move {
6275            loop {
6276                let (mut stream, _) = listener.accept().await.unwrap();
6277                let mut request = [0u8; 2048];
6278                let count = stream.read(&mut request).await.unwrap();
6279                assert!(count > 0, "a probe must send an HTTP request");
6280                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6281                let body = if code == 200 {
6282                    "ready"
6283                } else {
6284                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6285                };
6286                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6287                let _ = stream.write_all(response.as_bytes()).await;
6288            }
6289        });
6290        let configured = http_fixture(&dir, &url, 1000);
6291        let module =
6292            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6293                .supervise_configured_with_health(
6294                    configured.module_spec(),
6295                    true,
6296                    configured.health,
6297                    configured.drain_timeout_ms,
6298                    configured.restart,
6299                )
6300                .unwrap();
6301        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6302        status.store(503, std::sync::atomic::Ordering::SeqCst);
6303        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6304        assert!(module
6305            .status()
6306            .unwrap()
6307            .health
6308            .detail
6309            .unwrap()
6310            .contains("scratch failure"));
6311        status.store(200, std::sync::atomic::Ordering::SeqCst);
6312        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6313        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6314        let before = module.status().unwrap();
6315        let (spec, mut health) = module.configuration().unwrap();
6316        health.http = None;
6317        module
6318            .update_configuration(spec.clone(), health.clone(), Some(10))
6319            .await
6320            .unwrap();
6321        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6322        health.http = Some(url);
6323        module
6324            .update_configuration(spec, health, Some(10))
6325            .await
6326            .unwrap();
6327        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6328        assert_eq!(
6329            module.status().unwrap().pid,
6330            before.pid,
6331            "changing a probe must apply live, not restart its process"
6332        );
6333        let (spec, mut health) = module.configuration().unwrap();
6334        health.failure_threshold = 2;
6335        module
6336            .update_configuration(spec, health, Some(10))
6337            .await
6338            .unwrap();
6339        status.store(503, std::sync::atomic::Ordering::SeqCst);
6340        timeout(Duration::from_secs(5), async {
6341            while module.status().unwrap().spawn_generation == before.spawn_generation {
6342                sleep(Duration::from_millis(5)).await;
6343            }
6344        })
6345        .await
6346        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6347        module.drain().await.unwrap();
6348        server.abort();
6349    }
6350
6351    #[cfg(unix)]
6352    #[tokio::test]
6353    async fn http_health_replacement_recovers_after_one_restart() {
6354        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6355        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6356        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6357        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6358        let mut configured = http_fixture(&dir, &url, 2);
6359        configured.restart.max_restarts = 2;
6360        let module =
6361            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6362                .supervise_configured_with_health(
6363                    configured.module_spec(),
6364                    true,
6365                    configured.health,
6366                    configured.drain_timeout_ms,
6367                    configured.restart,
6368                )
6369                .unwrap();
6370        let first_generation = module.status().unwrap().spawn_generation;
6371        let observed = module.clone();
6372        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6373        let server = tokio::spawn(async move {
6374            let mut generation = first_generation;
6375            let mut probes = 0;
6376            loop {
6377                let (mut stream, _) = listener.accept().await.unwrap();
6378                let mut request = [0; 2048];
6379                assert!(stream.read(&mut request).await.unwrap() > 0);
6380                let current = observed.status().unwrap().spawn_generation;
6381                if current != generation {
6382                    generation = current;
6383                    probes = 0;
6384                }
6385                probes += 1;
6386                // The original process never recovers. Each replacement has one
6387                // transient miss, then answers successfully for its remaining life.
6388                let healthy = generation != first_generation && probes > 1;
6389                let code = if healthy { 200 } else { 503 };
6390                let response = format!(
6391                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6392                );
6393                let _ = stream.write_all(response.as_bytes()).await;
6394                if healthy {
6395                    healthy_tx.send(()).await.unwrap();
6396                }
6397            }
6398        });
6399        timeout(Duration::from_secs(5), async {
6400            let mut healthy_probes = 0;
6401            loop {
6402                if module.state().unwrap() == ModuleState::Failed {
6403                    break;
6404                }
6405                while healthy_rx.try_recv().is_ok() {
6406                    healthy_probes += 1;
6407                }
6408                if healthy_probes >= 3
6409                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6410                {
6411                    break;
6412                }
6413                sleep(Duration::from_millis(5)).await;
6414            }
6415        })
6416        .await
6417        .expect("the replacement must settle healthy without exhausting its restart budget");
6418        let status = module.status().unwrap();
6419        module.drain().await.unwrap();
6420        server.abort();
6421        assert_eq!(status.state, ModuleState::Running);
6422        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6423        assert_eq!(status.health.consecutive_failures, 0);
6424        assert_eq!(status.lifetime_restarts, 1);
6425        assert_eq!(status.restart_count, 1);
6426    }
6427
6428    #[cfg(unix)]
6429    #[tokio::test]
6430    async fn http_health_sustained_failure_exhausts_restart_budget() {
6431        use std::collections::BTreeMap;
6432        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6433        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6434        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6435        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6436        let mut configured = http_fixture(&dir, &url, 2);
6437        configured.restart.max_restarts = 2;
6438        let module =
6439            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6440                .supervise_configured_with_health(
6441                    configured.module_spec(),
6442                    true,
6443                    configured.health,
6444                    configured.drain_timeout_ms,
6445                    configured.restart,
6446                )
6447                .unwrap();
6448        let observed = module.clone();
6449        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6450        let served = probes.clone();
6451        let server = tokio::spawn(async move {
6452            loop {
6453                let (mut stream, _) = listener.accept().await.unwrap();
6454                let mut request = [0; 2048];
6455                assert!(stream.read(&mut request).await.unwrap() > 0);
6456                *served
6457                    .lock()
6458                    .unwrap()
6459                    .entry(observed.status().unwrap().spawn_generation)
6460                    .or_default() += 1;
6461                let _ = stream
6462                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6463                    .await;
6464            }
6465        });
6466        timeout(Duration::from_secs(5), async {
6467            while module.state().unwrap() != ModuleState::Failed {
6468                sleep(Duration::from_millis(5)).await;
6469            }
6470        })
6471        .await
6472        .expect("sustained failures must still exhaust the health restart budget");
6473        let status = module.status().unwrap();
6474        server.abort();
6475        assert_eq!(status.restart_count, 2);
6476        assert_eq!(status.lifetime_restarts, 2);
6477        assert!(!status.process_alive);
6478        assert_eq!(
6479            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6480            vec![2, 2, 2]
6481        );
6482    }
6483
6484    #[cfg(unix)]
6485    #[tokio::test]
6486    async fn spawn_clears_the_previous_process_health() {
6487        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6488        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6489        configured.health.cadence = Duration::from_secs(3600);
6490        let module =
6491            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6492                .supervise_configured_with_health(
6493                    configured.module_spec(),
6494                    false,
6495                    configured.health,
6496                    configured.drain_timeout_ms,
6497                    configured.restart,
6498                )
6499                .unwrap();
6500        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6501            state.health.status = SupervisorHealthStatus::Unresponsive;
6502            state.health.consecutive_failures = 3;
6503            state.health.detail = Some("old process refused connections".into());
6504            state.health.metrics = Some(serde_json::json!({"old": true}));
6505            state.health.last_probe_ms = Some(123);
6506            state.health.last_action = Some("restart".into());
6507            state.health.last_action_ms = Some(124);
6508        })
6509        .unwrap();
6510        module.set_enabled(true).await.unwrap();
6511        let health = module.status().unwrap().health;
6512        module.drain().await.unwrap();
6513        assert_eq!(health.consecutive_failures, 0);
6514        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6515        assert_eq!(health.detail, None);
6516        assert_eq!(health.metrics, None);
6517        assert_eq!(health.last_probe_ms, None);
6518        assert_eq!(health.last_action.as_deref(), Some("restart"));
6519        assert_eq!(health.last_action_ms, Some(124));
6520    }
6521
6522    #[cfg(unix)]
6523    #[tokio::test]
6524    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6525        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6526        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6527        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6528        drop(unused);
6529        let configured = http_fixture(&dir, &url, 2);
6530        let module =
6531            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6532                .supervise_configured_with_health(
6533                    configured.module_spec(),
6534                    true,
6535                    configured.health,
6536                    configured.drain_timeout_ms,
6537                    configured.restart,
6538                )
6539                .unwrap();
6540        let before = module.status().unwrap().spawn_generation;
6541        timeout(Duration::from_secs(5), async {
6542            loop {
6543                let status = module.status().unwrap();
6544                if status.spawn_generation > before {
6545                    assert!(status.lifetime_restarts > 0);
6546                    break;
6547                }
6548                sleep(Duration::from_millis(5)).await;
6549            }
6550        })
6551        .await
6552        .expect("sustained HTTP refusal must trigger the health restart policy");
6553        module.drain().await.unwrap();
6554    }
6555
6556    #[cfg(unix)]
6557    #[tokio::test]
6558    async fn http_health_timeout_honours_deadline() {
6559        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6560        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6561        let server = tokio::spawn(async move {
6562            let _held = listener.accept().await.unwrap();
6563            std::future::pending::<()>().await;
6564        });
6565        let error = timeout(
6566            Duration::from_secs(1),
6567            probe_http_health(&url, Duration::from_millis(10)),
6568        )
6569        .await
6570        .expect("the probe must enforce its own deadline")
6571        .unwrap_err();
6572        server.abort();
6573        assert!(error.to_string().contains("timed out"));
6574    }
6575
6576    #[cfg(unix)]
6577    #[tokio::test]
6578    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6579        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6580        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6581        let url = format!(
6582            "http://localhost:{}/healthz",
6583            listener.local_addr().unwrap().port()
6584        );
6585        let server = tokio::spawn(async move {
6586            let (mut stream, _) = listener.accept().await.unwrap();
6587            let mut request = [0u8; 2048];
6588            assert!(stream.read(&mut request).await.unwrap() > 0);
6589            stream
6590                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6591                .await
6592                .unwrap();
6593        });
6594        // The deadline only bounds a hang. A probe that never tried the IPv6
6595        // address would be refused on 127.0.0.1 and fail at once, so a longer
6596        // deadline does not weaken the assertion; one second timed out under a
6597        // loaded parallel test run.
6598        assert_eq!(
6599            probe_http_health(&url, Duration::from_secs(10))
6600                .await
6601                .unwrap()
6602                .status,
6603            HealthStatus::Ok
6604        );
6605        server.await.unwrap();
6606    }
6607
6608    #[cfg(unix)]
6609    #[tokio::test]
6610    async fn http_health_timeout_keeps_partial_status_and_body() {
6611        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6612        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6613        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6614        let server = tokio::spawn(async move {
6615            let (mut stream, _) = listener.accept().await.unwrap();
6616            let mut request = [0u8; 2048];
6617            assert!(stream.read(&mut request).await.unwrap() > 0);
6618            stream
6619                .write_all(
6620                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6621                )
6622                .await
6623                .unwrap();
6624            std::future::pending::<()>().await;
6625        });
6626        let error = probe_http_health(&url, Duration::from_secs(1))
6627            .await
6628            .unwrap_err()
6629            .to_string();
6630        server.abort();
6631        assert!(
6632            error.contains("timed out")
6633                && error.contains("503 Unavailable")
6634                && error.contains("partial diagnostic"),
6635            "{error}"
6636        );
6637    }
6638
6639    #[cfg(unix)]
6640    #[tokio::test]
6641    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6642        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6643        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6644        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6645        let server = tokio::spawn(async move {
6646            let (mut stream, _) = listener.accept().await.unwrap();
6647            let mut request = [0u8; 2048];
6648            assert!(stream.read(&mut request).await.unwrap() > 0);
6649            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6650            let response = format!(
6651                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6652                body.len()
6653            );
6654            stream.write_all(response.as_bytes()).await.unwrap();
6655        });
6656        let error = probe_http_health(&url, Duration::from_secs(1))
6657            .await
6658            .unwrap_err()
6659            .to_string();
6660        server.await.unwrap();
6661        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6662        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6663        assert!(!error.contains("not-in-diagnostic"));
6664    }
6665
6666    #[cfg(unix)]
6667    #[tokio::test]
6668    async fn http_health_real_nats_server_monitoring() {
6669        if std::process::Command::new("nats-server")
6670            .arg("--version")
6671            .env("XDG_DATA_HOME", std::env::temp_dir())
6672            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6673            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6674            .output()
6675            .is_err()
6676        {
6677            eprintln!(
6678                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6679            );
6680            return;
6681        }
6682        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6683        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6684        let port = monitor.local_addr().unwrap().port();
6685        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6686        let client_port = client.local_addr().unwrap().port();
6687        let config = dir.join("server.conf");
6688        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6689        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6690        configured.program = PathBuf::from("nats-server");
6691        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6692        drop(monitor);
6693        drop(client);
6694        let module =
6695            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6696                .supervise_configured_with_health(
6697                    configured.module_spec(),
6698                    true,
6699                    configured.health,
6700                    configured.drain_timeout_ms,
6701                    configured.restart,
6702                )
6703                .unwrap();
6704        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6705        module.drain().await.unwrap();
6706    }
6707
6708    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6709    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6710        let supervisor =
6711            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6712        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6713        let initial = ModuleSpec {
6714            module_id: "rescan-preserves-spawn-facts".to_string(),
6715            program: PathBuf::from("/spawned/module"),
6716            args: Vec::new(),
6717            env: Vec::new(),
6718            reserved: false,
6719            reserved_prefixes: Vec::new(),
6720            protocol: ModuleProtocol::Subc,
6721            overlap: Default::default(),
6722        };
6723        let module = supervisor.supervised_module(
6724            initial.clone(),
6725            supervisor.runtime_config(),
6726            snapshot,
6727            None,
6728        );
6729        let before = module.status().unwrap();
6730        let mut replacement = initial;
6731        replacement.program = PathBuf::from("/rescanned/replacement-module");
6732
6733        module
6734            .update_configuration(replacement, HealthConfig::default(), None)
6735            .await
6736            .unwrap();
6737
6738        let after = module.status().unwrap();
6739        assert_eq!(after.pid, before.pid);
6740        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6741        assert_eq!(after.spawned_from, before.spawned_from);
6742        drop(module);
6743    }
6744}
6745
6746fn unix_ms_now() -> u64 {
6747    SystemTime::now()
6748        .duration_since(UNIX_EPOCH)
6749        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6750        .unwrap_or(0)
6751}
6752
6753async fn supervise_loop(
6754    mut spec: ModuleSpec,
6755    mut runtime: SupervisorRuntimeConfig,
6756    registry: Arc<Registry>,
6757    process_liveness: Arc<SupervisorProcessLiveness>,
6758    snapshot: SharedSnapshot,
6759    mut child: Option<SupervisedChild>,
6760    mut commands: mpsc::Receiver<SupervisorCommand>,
6761) {
6762    let mut health_probe = HealthProbeRuntime::default();
6763    // Registry writes (including embedded callers) notify this module only.
6764    // Subscribe before the first refresh; watch retains changes that arrive
6765    // while commands or probes are running. No polling fallback is needed.
6766    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6767        Ok(changes) => Some(changes),
6768        Err(err) => {
6769            // A poisoned registry cannot accept further writes, so it cannot
6770            // recover via a timer. Keep exit and command handling alive.
6771            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6772            None
6773        }
6774    };
6775    // All restart backoffs run here, including health and operator requests.
6776    // While one is pending the loop serves commands, so disable or drain can
6777    // cancel the replacement without spawning a process just to stop it.
6778    let mut pending_respawn: Option<PendingRespawn> = None;
6779    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6780    // before anything else so a stop that interrupted a swap runs at once.
6781    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6782    let mut previous_instance_recheck = tokio::time::interval_at(
6783        Instant::now() + Duration::from_secs(5),
6784        Duration::from_secs(5),
6785    );
6786    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6787    loop {
6788        #[cfg(test)]
6789        {
6790            lock_snapshot(&snapshot).unwrap().actor_select = None;
6791        }
6792        #[cfg(target_os = "macos")]
6793        if let Some(active) = child.as_mut() {
6794            active.confirm_privacy_exec().await;
6795        }
6796        if let Some(scheduled) = runtime
6797            .scheduled_respawn
6798            .lock()
6799            .unwrap_or_else(|p| p.into_inner())
6800            .take()
6801        {
6802            runtime.recovery_is_daemon = scheduled.daemon_initiated;
6803            pending_respawn = Some(scheduled);
6804        }
6805        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6806            pending_respawn = None;
6807            cancel_deferred_reload(
6808                &runtime,
6809                &spec.module_id,
6810                "respawn cancelled by a supervisor command",
6811            );
6812        }
6813        if child.is_none()
6814            && pending_respawn.is_none()
6815            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6816        {
6817            cancel_deferred_reload(
6818                &runtime,
6819                &spec.module_id,
6820                "respawn cancelled before a replacement was spawned",
6821            );
6822            let mut stranded = false;
6823            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6824                state.respawn_pending = false;
6825                state.coalesced_restart_pending = false;
6826                if matches!(
6827                    state.state,
6828                    ModuleState::Restarting
6829                        | ModuleState::Starting
6830                        | ModuleState::Draining
6831                        | ModuleState::Unresponsive
6832                ) {
6833                    error!(module_id = %spec.module_id, state = ?state.state,
6834                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6835                    stranded = true;
6836                }
6837            });
6838            if stranded {
6839                fail_snapshot(
6840                    &snapshot,
6841                    Some(&spec.module_id),
6842                    None,
6843                    &runtime.failure_context(runtime.recovery_is_daemon),
6844                );
6845            }
6846        }
6847        if let Some(command) = requeued.pop_front() {
6848            if !handle_supervisor_command(
6849                command,
6850                &mut spec,
6851                &mut runtime,
6852                &registry,
6853                &process_liveness,
6854                &snapshot,
6855                &mut child,
6856                &mut commands,
6857                &mut requeued,
6858            )
6859            .await
6860            {
6861                return;
6862            }
6863            if child.is_some() || !respawn_still_pending(&snapshot) {
6864                pending_respawn = None;
6865                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6866                    state.respawn_pending = false
6867                });
6868            }
6869            continue;
6870        }
6871        if child.is_some() {
6872            runtime.recovery_is_daemon = true;
6873            #[cfg(test)]
6874            {
6875                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6876            }
6877            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6878            let active = child.as_mut().expect("child checked above");
6879            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6880                active.registration_deadline = None;
6881            }
6882            let registration_deadline = active.registration_deadline;
6883            let no_checks = runtime.health.http.is_none()
6884                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6885                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6886            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6887                state
6888                    .failed_retry
6889                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
6890            });
6891            let wake_at = match (health_probe.wake_at(), uptime_reset) {
6892                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6893                (probe, None) => probe,
6894            };
6895            let wake_at = match (wake_at, registration_deadline) {
6896                (Some(wake), Some(register)) => Some(wake.min(register)),
6897                (wake, register) => wake.or(register),
6898            };
6899            #[cfg(test)]
6900            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
6901            let probe_sleep = supervisor_sleep(wake_at);
6902            tokio::pin!(probe_sleep);
6903            // A `protocol: "none"` child (a plain process that never registers
6904            // over the subc wire, such as nats-server) with no HTTP health
6905            // check has nothing to probe, so it waits only for its exit or a
6906            // supervisor command. A rescan that adds an HTTP check is a command.
6907            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6908            #[cfg(test)]
6909            {
6910                let registration_pending = wire_child
6911                    && registration_changes
6912                        .as_ref()
6913                        .is_some_and(|changes| changes.has_changed().unwrap());
6914                let mut state = lock_snapshot(&snapshot).unwrap();
6915                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6916                    generation: state.spawn_generation,
6917                    turn: state.actor_turns,
6918                    registered_connection: health_probe.registered_connection,
6919                    next_probe_at: health_probe.next_probe_at,
6920                    wake_after,
6921                });
6922            }
6923            let active_child = child.as_mut().expect("child checked above");
6924            tokio::select! {
6925                wait_result = active_child.wait() => {
6926                    if let Ok(mut state) = lock_snapshot(&snapshot) {
6927                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
6928                    }
6929                    #[cfg(test)]
6930                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6931                    // Every arm below that gives up on the CHILD must keep the
6932                    // supervision task itself alive (child = None, loop
6933                    // continues into command-serving mode). Returning here
6934                    // closes the command channel, which makes the module
6935                    // permanently unrestartable in-band: a clean child exit
6936                    // of an enabled module once wedged the fleet this way
6937                    // ('supervisor command channel is closed') and required a
6938                    // full daemon restart to recover.
6939                    let exit_report = match wait_result {
6940                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6941                        Err(err) => {
6942                            active_child.drain_stderr(&spec.module_id).await;
6943                            fail_after_wait_error(
6944                                &spec.module_id,
6945                                &snapshot,
6946                                &runtime.terminal_ring,
6947                                &runtime.spawn_events,
6948                                format!("failed to wait for supervised module: {err}"),
6949                            &runtime.failure_context(true),
6950                            );
6951                            untrack_if_registration_released(
6952                                &process_liveness,
6953                                &registry,
6954                                &spec.module_id,
6955                                &snapshot,
6956                            );
6957                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6958                            child = None;
6959                            continue;
6960                        }
6961                    };
6962                    active_child.drain_stderr(&spec.module_id).await;
6963
6964                    let next = on_child_exit(
6965                        &spec,
6966                        runtime.restart_policy,
6967                        &registry,
6968                        &snapshot,
6969                        &runtime.terminal_ring,
6970                        &runtime.spawn_events,
6971                        &runtime.child_roster,
6972                        exit_report,
6973                        true,
6974                    ).await;
6975                    runtime.start_requester = None;
6976                    runtime.pending_activity_restart = false;
6977                    // The exit is recorded, so a daemon shutdown may stop
6978                    // waiting for this child (see `SupervisedChild::wait`).
6979                    active_child.release_roster();
6980                    match next {
6981                        NextAction::Stop { registration_released } => {
6982                            if registration_released {
6983                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6984                            }
6985                            child = None;
6986                        }
6987                        NextAction::Restart { schedule } => {
6988                            let delay = schedule.map_or(
6989                                runtime.restart_policy.delay_for_restart(0),
6990                                |schedule| schedule.delay,
6991                            );
6992                            if let Some(schedule) = schedule {
6993                                log_crash_respawn(&spec.module_id, schedule);
6994                            }
6995                            // The exited child is fully recorded at this point,
6996                            // so release it and count the backoff down in the
6997                            // command-serving branch below rather than sleeping
6998                            // here: commands cannot be received from inside this
6999                            // select arm, and an operator disable or drain that
7000                            // arrives during the backoff must cancel the pending
7001                            // respawn instead of waiting for it to spawn first.
7002                            child = None;
7003                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7004                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7005                        }
7006                    }
7007                }
7008                command = commands.recv() => {
7009                    #[cfg(test)]
7010                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7011                    let Some(command) = command else {
7012                        return;
7013                    };
7014                    if !handle_supervisor_command(
7015                        command,
7016                        &mut spec,
7017                        &mut runtime,
7018                        &registry,
7019                        &process_liveness,
7020                        &snapshot,
7021                        &mut child,
7022                        &mut commands,
7023                        &mut requeued,
7024                    ).await {
7025                        return;
7026                    }
7027                }
7028                _ = async {
7029                    match registration_changes.as_mut() {
7030                        Some(changes) => { let _ = changes.changed().await; }
7031                        None => std::future::pending().await,
7032                    }
7033                }, if wire_child => {
7034                    #[cfg(test)]
7035                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7036                }
7037                _ = &mut probe_sleep => {
7038                    #[cfg(test)]
7039                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7040                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7041                        // Recheck after the wake: HELLO, exit or shutdown may
7042                        // have become ready in the same executor tick.
7043                        active_child.registration_deadline = None;
7044                        if !runtime.child_roster.is_closed()
7045                            && !active_child.registered_since_spawn()
7046                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7047                        {
7048                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7049                                wait_secs = runtime.register_timeout.as_secs(),
7050                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7051                            active_child.spawn_failure = Some("registration_timeout".into());
7052                            if let Err(error) = stop_unregistered_child(active_child).await {
7053                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7054                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7055                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7056                                    &runtime.failure_context(true));
7057                                child = None;
7058                            }
7059                            // The ordinary exit handler records the child's exit,
7060                            // spends crash budget, and schedules recovery.
7061                            continue;
7062                        }
7063                    }
7064                    if health_probe.due() {
7065                        run_health_probe_cycle(
7066                            &spec,
7067                            &runtime,
7068                            &registry,
7069                            &process_liveness,
7070                            &snapshot,
7071                            &mut child,
7072                        ).await;
7073                        if child.is_none() {
7074                            runtime.start_requester = None;
7075                            runtime.pending_activity_restart = false;
7076                        }
7077                        if child.is_some() {
7078                            health_probe.schedule_next(&spec, runtime.health.cadence);
7079                        }
7080                    }
7081                }
7082            }
7083        } else if let Some(pending) = pending_respawn {
7084            tokio::select! {
7085                _ = sleep_until(pending.deadline) => {
7086                    pending_respawn = None;
7087                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7088                    // A command handled below while the backoff elapsed may
7089                    // have stopped the module; never respawn past an operator's
7090                    // disable or drain.
7091                    if !respawn_still_pending(&snapshot) {
7092                        continue;
7093                    }
7094                    // The daemon began shutting down during the backoff: the
7095                    // spawn would be refused anyway, and refusing it here
7096                    // leaves the module stopped instead of reporting a
7097                    // failed restart.
7098                    if runtime.child_roster.is_closed() {
7099                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7100                            state.state = ModuleState::Stopped;
7101                        });
7102                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7103                        continue;
7104                    }
7105                    if let Err(err) = release_dead_registration(
7106                        &registry,
7107                        runtime.forwarding.as_deref(),
7108                        &snapshot,
7109                        &spec.module_id,
7110                        &runtime.failure_context(runtime.recovery_is_daemon),
7111                    ).await {
7112                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7113                        continue;
7114                    }
7115
7116                    if matches!(pending.kind, RespawnKind::Reload) {
7117                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7118                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7119                        if result.is_ok() && runtime.pending_activity_restart {
7120                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7121                        }
7122                        runtime.pending_activity_restart = false;
7123                        if let Some(reply) = reply { let _ = reply.send(result); }
7124                        continue;
7125                    }
7126                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7127                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7128                        Ok(next_child) => {
7129                            child = Some(next_child);
7130                            if runtime.pending_activity_restart {
7131                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7132                                runtime.pending_activity_restart = false;
7133                            }
7134                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7135                        }
7136                        Err(err) => {
7137                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7138                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7139                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7140                        }
7141                    }
7142                }
7143                command = commands.recv() => {
7144                    let Some(command) = command else {
7145                        return;
7146                    };
7147                    if !handle_supervisor_command(
7148                        command,
7149                        &mut spec,
7150                        &mut runtime,
7151                        &registry,
7152                        &process_liveness,
7153                        &snapshot,
7154                        &mut child,
7155                        &mut commands,
7156                        &mut requeued,
7157                    ).await {
7158                        return;
7159                    }
7160                    // Reconcile the pending respawn with what the command did:
7161                    // a start may already have spawned a fresh child,
7162                    // while a disable or drain moved the snapshot out of the
7163                    // state the respawn was counting down from.
7164                    if child.is_some() || !respawn_still_pending(&snapshot) {
7165                        pending_respawn = None;
7166                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7167                    }
7168                }
7169            }
7170        } else {
7171            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7172                (state.enabled && state.state == ModuleState::Failed)
7173                    .then_some(state.failed_retry.pending)
7174                    .flatten()
7175            });
7176            let command = if lock_snapshot(&snapshot)
7177                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7178            {
7179                tokio::select! {
7180                    command = commands.recv() => command,
7181                    _ = previous_instance_recheck.tick() => {
7182                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7183                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7184                            if enabled && !runtime.child_roster.is_closed() {
7185                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7186                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7187                                    Ok(next_child) => child = Some(next_child),
7188                                    Err(error) => {
7189                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7190                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7191                                    }
7192                                }
7193                            } else {
7194                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7195                                    state.waiting_for_previous_pid = None;
7196                                    state.waiting_for_previous_detail = None;
7197                                });
7198                            }
7199                        }
7200                        continue;
7201                    }
7202                }
7203            } else {
7204                tokio::select! {
7205                    biased;
7206                    command = commands.recv() => command,
7207                    _ = async {
7208                        match slow_retry {
7209                            Some(retry) => sleep_until(retry.deadline).await,
7210                            None => std::future::pending().await,
7211                        }
7212                    } => {
7213                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7214                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7215                            state.failed_retry.pending = None;
7216                        });
7217                        if runtime.child_roster.is_closed() {
7218                            continue;
7219                        }
7220                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7221                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7222                        if let Err(error) = set_child_enabled(
7223                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7224                            &runtime.failure_context(true),
7225                        ).await {
7226                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7227                        }
7228                        continue;
7229                    }
7230                }
7231            };
7232            let Some(command) = command else {
7233                return;
7234            };
7235            if !handle_supervisor_command(
7236                command,
7237                &mut spec,
7238                &mut runtime,
7239                &registry,
7240                &process_liveness,
7241                &snapshot,
7242                &mut child,
7243                &mut commands,
7244                &mut requeued,
7245            )
7246            .await
7247            {
7248                return;
7249            }
7250        }
7251    }
7252}
7253
7254// Preserve the deadline until the future is polled. Starting a relative sleep
7255// after the executor yields would restart part of the countdown.
7256async fn supervisor_sleep(wake_at: Option<Instant>) {
7257    match wake_at {
7258        Some(deadline) => sleep_until(deadline).await,
7259        None => std::future::pending().await,
7260    }
7261}
7262
7263fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7264    info!(
7265        module_id,
7266        restart_in_window = schedule.restart_in_window,
7267        delay_ms = schedule.delay.as_millis() as u64,
7268        "respawning after crash"
7269    );
7270}
7271
7272/// Whether the respawn a backoff was counting down to is still wanted. A
7273/// disable or drain handled while the backoff elapsed moves the snapshot out
7274/// of `Restarting`, and the operator's stop must win over the pending respawn,
7275/// so every sleep-then-spawn path re-validates against the live snapshot
7276/// instead of assuming the state it left behind still holds.
7277fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7278    matches!(
7279        lock_snapshot(snapshot),
7280        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7281    )
7282}
7283
7284enum NextAction {
7285    Stop {
7286        registration_released: bool,
7287    },
7288    Restart {
7289        schedule: Option<CrashRestartSchedule>,
7290    },
7291}
7292
7293#[allow(clippy::too_many_arguments)]
7294async fn handle_supervisor_command(
7295    command: SupervisorCommand,
7296    spec: &mut ModuleSpec,
7297    runtime: &mut SupervisorRuntimeConfig,
7298    registry: &Arc<Registry>,
7299    process_liveness: &SupervisorProcessLiveness,
7300    snapshot: &SharedSnapshot,
7301    child: &mut Option<SupervisedChild>,
7302    commands: &mut mpsc::Receiver<SupervisorCommand>,
7303    requeued: &mut VecDeque<SupervisorCommand>,
7304) -> bool {
7305    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7306        runtime.recovery_is_daemon = false;
7307    }
7308    if matches!(
7309        command,
7310        SupervisorCommand::Drain { .. }
7311            | SupervisorCommand::Retire { .. }
7312            | SupervisorCommand::SetEnabled { .. }
7313            | SupervisorCommand::Restart { .. }
7314    ) {
7315        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7316            state.failed_retry.cancel()
7317        });
7318    }
7319    let command =
7320        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7321            match command {
7322                SupervisorCommand::Restart { reply, .. }
7323                | SupervisorCommand::Reload { reply, .. }
7324                | SupervisorCommand::Swap { reply, .. } => {
7325                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7326                    return true;
7327                }
7328                SupervisorCommand::SetEnabled {
7329                    enabled: true,
7330                    reply,
7331                    ..
7332                } => {
7333                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7334                    return true;
7335                }
7336                other => other,
7337            }
7338        } else {
7339            command
7340        };
7341    match command {
7342        SupervisorCommand::Drain { requester, reply } => {
7343            // A plain stop runs no forwarding drain, so nothing reaches the
7344            // module over its connection before the wait: ask by signal.
7345            let result = drain_optional_child(
7346                &spec.module_id,
7347                spec.protocol,
7348                StopNotice::NotSent,
7349                registry,
7350                runtime.forwarding.as_deref(),
7351                snapshot,
7352                &runtime.terminal_ring,
7353                &runtime.spawn_events,
7354                child,
7355                runtime.drain_timeout,
7356                ModuleState::Stopped,
7357                None,
7358                &FailureContext::operator(),
7359            )
7360            .await;
7361            let registration_released = result.is_ok();
7362            if registration_released {
7363                runtime.record_activity_transition(
7364                    "subc.module_stopped",
7365                    &spec.module_id,
7366                    requester,
7367                );
7368            }
7369            let _ = reply.send(result);
7370            if registration_released {
7371                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7372            }
7373            false
7374        }
7375        SupervisorCommand::Retire { requester, reply } => {
7376            let result = async {
7377                let stop_notice = begin_forwarding_drain_if_configured(
7378                    spec,
7379                    runtime,
7380                    registry,
7381                    snapshot,
7382                    None,
7383                    RouteCloseReason::Disable,
7384                )
7385                .await?;
7386                drain_optional_child(
7387                    &spec.module_id,
7388                    spec.protocol,
7389                    stop_notice,
7390                    registry,
7391                    runtime.forwarding.as_deref(),
7392                    snapshot,
7393                    &runtime.terminal_ring,
7394                    &runtime.spawn_events,
7395                    child,
7396                    runtime.drain_timeout,
7397                    ModuleState::Stopped,
7398                    None,
7399                    &FailureContext::operator(),
7400                )
7401                .await
7402            }
7403            .await;
7404            let registration_released = result.is_ok();
7405            if registration_released {
7406                runtime.record_activity_transition(
7407                    "subc.module_stopped",
7408                    &spec.module_id,
7409                    requester,
7410                );
7411            }
7412            let _ = reply.send(result);
7413            if registration_released {
7414                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7415            }
7416            false
7417        }
7418        SupervisorCommand::Restart {
7419            requester,
7420            drain_timeout_ms,
7421            received_at_generation,
7422            queued_at,
7423            reply,
7424        } => {
7425            // Without this line a restart that waited in the queue (behind a
7426            // health probe cycle or another command) was invisible: the log
7427            // showed only the drain timing out, minutes after the operator's call.
7428            info!(
7429                module_id = %spec.module_id,
7430                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7431                "restart command dequeued"
7432            );
7433            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7434            // caller whose own request lane rides the module being restarted: the
7435            // caller's in-flight request keeps the drain from quiescing, the drain
7436            // keeps the restart from completing, and the completion keeps the reply
7437            // from releasing the caller — so the drain always timed out and cut the
7438            // initiator with a GOODBYE, even on a healthy module. Replying once the
7439            // restart is validated lets a self-lane caller settle, which is exactly
7440            // what makes the drain succeed. Completion is observable via
7441            // supervisor.list / module status; a post-ack failure lands the module
7442            // in a visible terminal state below rather than in a reply nobody can
7443            // receive.
7444            let validation = match lock_snapshot(snapshot) {
7445                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7446                    module_id: spec.module_id.clone(),
7447                }),
7448                Ok(_) => Ok(()),
7449                Err(err) => Err(err),
7450            };
7451            let initiated = validation.is_ok();
7452            let _ = reply.send(validation);
7453            // A restart asks for a fresh process. Commands run one at a time,
7454            // so a restart queued behind another restart (two operator calls
7455            // in quick succession) is dequeued the moment the first one has
7456            // spawned its replacement -- before that process has sent HELLO.
7457            // Running it would drain and kill the process the first restart
7458            // just produced, which is the opposite of what both callers asked
7459            // for. If a process spawned after this request was received is
7460            // still supervised, the request is already satisfied. Not when the
7461            // configuration changed since that spawn: then the newer process
7462            // predates the spec this restart may exist to apply.
7463            // Generation publication can lag a child that is already serving
7464            // while its identity is recorded. Only process creation after request
7465            // receipt satisfies a restart, not publication after receipt.
7466            let satisfied_by_generation = if initiated
7467                && child
7468                    .as_ref()
7469                    .is_some_and(|child| child.spawned_at > queued_at)
7470            {
7471                lock_snapshot(snapshot).ok().and_then(|state| {
7472                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7473                })
7474            } else {
7475                None
7476            };
7477            let satisfied_by_pending = initiated
7478                && child.is_none()
7479                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7480                    // No child exists yet, so this replacement will necessarily
7481                    // be created after the queued request, unlike a published
7482                    // generation whose process may already have been serving.
7483                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7484                    if pending {
7485                        state.coalesced_restart_pending = true;
7486                    }
7487                    pending
7488                });
7489            if satisfied_by_pending {
7490                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7491            } else if let Some(generation) = satisfied_by_generation {
7492                info!(
7493                    module_id = %spec.module_id,
7494                    received_at_generation,
7495                    "restart already satisfied by generation {generation}; not restarting again"
7496                );
7497            } else if initiated {
7498                // Precedence: this restart's operator override, else the module's
7499                // configured budget (already resolved into the runtime).
7500                let drain_timeout = drain_timeout_ms
7501                    .map(Duration::from_millis)
7502                    .unwrap_or(runtime.drain_timeout);
7503                if let Err(err) = restart_child(
7504                    spec,
7505                    runtime,
7506                    registry,
7507                    process_liveness,
7508                    snapshot,
7509                    child,
7510                    drain_timeout,
7511                )
7512                .await
7513                {
7514                    warn!(
7515                        module_id = %spec.module_id,
7516                        error = %err,
7517                        "operator restart failed after initiation ack; module state carries the outcome"
7518                    );
7519                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7520                        state.state = ModuleState::Failed;
7521                        clear_current_process_facts(state);
7522                    });
7523                } else {
7524                    runtime.start_requester = requester;
7525                    runtime.pending_activity_restart = true;
7526                }
7527            }
7528            true
7529        }
7530        SupervisorCommand::Reload { requester, reply } => {
7531            let result =
7532                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7533            if result.is_ok() {
7534                runtime.start_requester = requester;
7535                runtime.pending_activity_restart = true;
7536            }
7537            if result.is_ok()
7538                && runtime
7539                    .scheduled_respawn
7540                    .lock()
7541                    .unwrap_or_else(|p| p.into_inner())
7542                    .as_ref()
7543                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7544            {
7545                *runtime
7546                    .deferred_reload_reply
7547                    .lock()
7548                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7549            } else {
7550                let _ = reply.send(result);
7551            }
7552            true
7553        }
7554        SupervisorCommand::SetEnabled {
7555            enabled,
7556            requester,
7557            reply,
7558        } => {
7559            let previous_requester = runtime.start_requester.clone();
7560            if enabled {
7561                runtime.start_requester = requester.clone();
7562            }
7563            let result = set_child_enabled(
7564                spec,
7565                runtime,
7566                registry,
7567                process_liveness,
7568                snapshot,
7569                child,
7570                enabled,
7571                &runtime.failure_context(false),
7572            )
7573            .await;
7574            if matches!(result, Ok(true)) {
7575                runtime.pending_activity_restart = false;
7576                if enabled {
7577                    runtime.start_requester = requester;
7578                } else {
7579                    runtime.start_requester = None;
7580                    runtime.record_activity_transition(
7581                        "subc.module_disabled",
7582                        &spec.module_id,
7583                        requester,
7584                    );
7585                }
7586            } else {
7587                runtime.start_requester = previous_requester;
7588            }
7589            let _ = reply.send(result);
7590            true
7591        }
7592        SupervisorCommand::UpdateConfiguration {
7593            spec: next_spec,
7594            health,
7595            drain_timeout_ms,
7596            reply,
7597        } => {
7598            if let Some(handle) = &runtime.supervisor_handle {
7599                handle.apply_identity_configuration(&next_spec);
7600            }
7601            *spec = next_spec;
7602            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7603                state.configuration_updated_since_spawn = true;
7604            });
7605            let health_changed = runtime.health != health;
7606            runtime.health = health;
7607            // Reset the cadence and old endpoint's failure streak on a live
7608            // health-policy change rather than waiting for its old deadline.
7609            if health_changed {
7610                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7611                    state.health = ModuleHealthStatus::default();
7612                });
7613            }
7614            runtime.drain_timeout = drain_timeout_ms
7615                .map(Duration::from_millis)
7616                .unwrap_or(runtime.default_drain_timeout);
7617            *runtime
7618                .effective_drain_timeout
7619                .lock()
7620                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7621            let _ = reply.send(());
7622            true
7623        }
7624        SupervisorCommand::Swap {
7625            requester,
7626            ready_timeout,
7627            reply,
7628        } => {
7629            let mut swap_runtime = runtime.clone();
7630            swap_runtime.start_requester = requester;
7631            let before = lock_snapshot(snapshot)
7632                .ok()
7633                .map(|state| state.spawn_generation);
7634            let end = swap::run_swap(
7635                spec,
7636                &swap_runtime,
7637                registry,
7638                process_liveness,
7639                snapshot,
7640                child,
7641                commands,
7642                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7643                reply,
7644            )
7645            .await;
7646            if before
7647                != lock_snapshot(snapshot)
7648                    .ok()
7649                    .map(|state| state.spawn_generation)
7650            {
7651                runtime.start_requester = swap_runtime.start_requester;
7652            }
7653            requeued.extend(end.requeue);
7654            true
7655        }
7656    }
7657}
7658
7659async fn restart_child(
7660    spec: &ModuleSpec,
7661    runtime: &SupervisorRuntimeConfig,
7662    registry: &Registry,
7663    process_liveness: &SupervisorProcessLiveness,
7664    snapshot: &SharedSnapshot,
7665    child: &mut Option<SupervisedChild>,
7666    drain_timeout: Duration,
7667) -> Result<(), SuperviseError> {
7668    // Restart cycles a running module; it must not silently start a disabled one.
7669    if !lock_snapshot(snapshot)?.enabled {
7670        return Err(SuperviseError::Disabled {
7671            module_id: spec.module_id.clone(),
7672        });
7673    }
7674    let stop_notice = begin_forwarding_drain_with_timeout(
7675        spec,
7676        runtime,
7677        registry,
7678        snapshot,
7679        None,
7680        RouteCloseReason::Restart,
7681        drain_timeout,
7682    )
7683    .await?;
7684
7685    if child.is_some() {
7686        drain_optional_child(
7687            &spec.module_id,
7688            spec.protocol,
7689            stop_notice,
7690            registry,
7691            runtime.forwarding.as_deref(),
7692            snapshot,
7693            &runtime.terminal_ring,
7694            &runtime.spawn_events,
7695            child,
7696            drain_timeout,
7697            ModuleState::Restarting,
7698            Some(true),
7699            &FailureContext::operator(),
7700        )
7701        .await?;
7702    } else {
7703        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7704            state.enabled = true;
7705            state.state = ModuleState::Restarting;
7706            clear_current_process_facts(state);
7707        })?;
7708        release_dead_registration(
7709            registry,
7710            runtime.forwarding.as_deref(),
7711            snapshot,
7712            &spec.module_id,
7713            &runtime.failure_context(false),
7714        )
7715        .await?;
7716    }
7717
7718    reset_restart_count(snapshot, &spec.module_id)?;
7719    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7720    schedule_respawn(
7721        runtime,
7722        snapshot,
7723        &spec.module_id,
7724        runtime.restart_policy.backoff,
7725        RespawnKind::Spawn,
7726        false,
7727    )
7728}
7729
7730async fn reload_child(
7731    spec: &ModuleSpec,
7732    runtime: &SupervisorRuntimeConfig,
7733    registry: &Registry,
7734    process_liveness: &SupervisorProcessLiveness,
7735    snapshot: &SharedSnapshot,
7736    child: &mut Option<SupervisedChild>,
7737) -> Result<(), SuperviseError> {
7738    // Reload cycles a running module; it must not silently start a disabled one.
7739    if !lock_snapshot(snapshot)?.enabled {
7740        return Err(SuperviseError::Disabled {
7741            module_id: spec.module_id.clone(),
7742        });
7743    }
7744    let stop_notice = begin_forwarding_drain(
7745        spec,
7746        runtime,
7747        registry,
7748        snapshot,
7749        Some(true),
7750        RouteCloseReason::Reload,
7751    )
7752    .await?;
7753
7754    if child.is_some() {
7755        drain_optional_child(
7756            &spec.module_id,
7757            spec.protocol,
7758            stop_notice,
7759            registry,
7760            runtime.forwarding.as_deref(),
7761            snapshot,
7762            &runtime.terminal_ring,
7763            &runtime.spawn_events,
7764            child,
7765            runtime.drain_timeout,
7766            ModuleState::Restarting,
7767            Some(true),
7768            &FailureContext::operator(),
7769        )
7770        .await?;
7771    } else {
7772        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7773            state.enabled = true;
7774            state.state = ModuleState::Restarting;
7775            clear_current_process_facts(state);
7776        })?;
7777        release_dead_registration(
7778            registry,
7779            runtime.forwarding.as_deref(),
7780            snapshot,
7781            &spec.module_id,
7782            &runtime.failure_context(false),
7783        )
7784        .await?;
7785    }
7786
7787    reset_restart_count(snapshot, &spec.module_id)?;
7788    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7789    schedule_respawn(
7790        runtime,
7791        snapshot,
7792        &spec.module_id,
7793        runtime.restart_policy.backoff,
7794        RespawnKind::Reload,
7795        false,
7796    )
7797}
7798
7799async fn finish_reload_child(
7800    spec: &ModuleSpec,
7801    runtime: &SupervisorRuntimeConfig,
7802    registry: &Registry,
7803    process_liveness: &SupervisorProcessLiveness,
7804    snapshot: &SharedSnapshot,
7805    child: &mut Option<SupervisedChild>,
7806) -> Result<(), SuperviseError> {
7807    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7808    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7809        Ok(next_child) => next_child,
7810        Err(err) => {
7811            return handle_reload_spawn_failure(
7812                spec,
7813                runtime,
7814                process_liveness,
7815                snapshot,
7816                child,
7817                format!("new child failed to spawn: {err}"),
7818            )
7819            .await;
7820        }
7821    };
7822    *child = Some(next_child);
7823
7824    let wait_outcome = {
7825        let active_child = child.as_mut().expect("new reload child was just stored");
7826        wait_for_registration_after_reload(
7827            registry,
7828            &spec.module_id,
7829            snapshot,
7830            active_child,
7831            REGISTRY_RELEASE_TIMEOUT,
7832        )
7833        .await?
7834    };
7835
7836    match wait_outcome {
7837        RegistrationWaitOutcome::Registered => {
7838            child
7839                .as_mut()
7840                .expect("registered reload child")
7841                .registration_deadline = None;
7842            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7843            Ok(())
7844        }
7845        RegistrationWaitOutcome::Exited(exit_report) => {
7846            if let Some(active_child) = child.as_mut() {
7847                active_child.drain_stderr(&spec.module_id).await;
7848            }
7849            // Keep the reaped child's roster guard until its terminal is written.
7850            // Shutdown waits on that guard, not on the child Option used for respawn.
7851            let mut exited_child = child.take().expect("exited reload child is still stored");
7852            #[cfg(test)]
7853            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7854                gate.reached.notify_one();
7855                gate.resume.notified().await;
7856            }
7857            let result = handle_reload_child_registration_failure(
7858                spec,
7859                runtime,
7860                registry,
7861                process_liveness,
7862                snapshot,
7863                child,
7864                ReloadRegistrationFailure {
7865                    exit_report: registration_failure_exit_report(exit_report),
7866                    reason: exited_child
7867                        .spawn_failure
7868                        .clone()
7869                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7870                },
7871            )
7872            .await;
7873            exited_child.release_roster();
7874            result
7875        }
7876        RegistrationWaitOutcome::TimedOut => {
7877            let mut timed_out_child = child
7878                .take()
7879                .expect("timed-out reload child is still running");
7880            timed_out_child
7881                .start_kill()
7882                .map_err(|source| SuperviseError::Kill {
7883                    module_id: spec.module_id.clone(),
7884                    source,
7885                })?;
7886            let status = timed_out_child
7887                .wait()
7888                .await
7889                .map_err(|source| SuperviseError::Wait {
7890                    module_id: spec.module_id.clone(),
7891                    source,
7892                })?;
7893            timed_out_child.drain_stderr(&spec.module_id).await;
7894            handle_reload_child_registration_failure(
7895                spec,
7896                runtime,
7897                registry,
7898                process_liveness,
7899                snapshot,
7900                child,
7901                ReloadRegistrationFailure {
7902                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7903                        snapshot,
7904                        &timed_out_child,
7905                        &status,
7906                    )),
7907                    reason: format!(
7908                        "new child did not register within {:?}",
7909                        REGISTRY_RELEASE_TIMEOUT
7910                    ),
7911                },
7912            )
7913            .await
7914        }
7915    }
7916}
7917
7918#[allow(clippy::too_many_arguments)]
7919async fn set_child_enabled(
7920    spec: &ModuleSpec,
7921    runtime: &SupervisorRuntimeConfig,
7922    registry: &Registry,
7923    process_liveness: &SupervisorProcessLiveness,
7924    snapshot: &SharedSnapshot,
7925    child: &mut Option<SupervisedChild>,
7926    enabled: bool,
7927    failure: &FailureContext,
7928) -> Result<bool, SuperviseError> {
7929    let (current_enabled, current_state, respawn_pending) = {
7930        let state = lock_snapshot(snapshot)?;
7931        (state.enabled, state.state, state.respawn_pending)
7932    };
7933    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7934    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7935    // clean (Stopped) has no live process and no other in-band recovery — the
7936    // operator's start is the explicit recovery act and resets the budget. Without
7937    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7938    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7939    // the one providing every agent's shell.
7940    let revive_terminal = enabled
7941        && current_enabled
7942        && child.is_none()
7943        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7944            || (current_state == ModuleState::Restarting && !respawn_pending));
7945    if current_enabled == enabled && !revive_terminal {
7946        return Ok(false);
7947    }
7948
7949    if enabled {
7950        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7951            state.enabled = true;
7952            state.state = ModuleState::Starting;
7953            clear_current_process_facts(state);
7954        })?;
7955        #[cfg(test)]
7956        if runtime.test_seed_stale_facts_before_enable_spawn {
7957            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7958                state.process_alive = true;
7959                state.pid = Some(41);
7960                state.spawned_at_ms = Some(42);
7961                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7962                state.spawned_file_identity = Some(SpawnedFileIdentity {
7963                    device: 43,
7964                    inode: 44,
7965                    #[cfg(windows)]
7966                    windows_image: None,
7967                });
7968            })?;
7969        }
7970        release_dead_registration(
7971            registry,
7972            runtime.forwarding.as_deref(),
7973            snapshot,
7974            &spec.module_id,
7975            failure,
7976        )
7977        .await?;
7978        reset_restart_count(snapshot, &spec.module_id)?;
7979        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7980        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7981            Ok(next_child) => next_child,
7982            Err(err) => {
7983                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7984                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7985                return Err(err);
7986            }
7987        };
7988        *child = Some(next_child);
7989        debug!(module_id = %spec.module_id, "supervised module enabled");
7990        Ok(true)
7991    } else {
7992        let stop_notice = begin_forwarding_drain_if_configured(
7993            spec,
7994            runtime,
7995            registry,
7996            snapshot,
7997            Some(false),
7998            RouteCloseReason::Disable,
7999        )
8000        .await?;
8001        drain_optional_child(
8002            &spec.module_id,
8003            spec.protocol,
8004            stop_notice,
8005            registry,
8006            runtime.forwarding.as_deref(),
8007            snapshot,
8008            &runtime.terminal_ring,
8009            &runtime.spawn_events,
8010            child,
8011            runtime.drain_timeout,
8012            ModuleState::Disabled,
8013            Some(false),
8014            failure,
8015        )
8016        .await?;
8017        debug!(module_id = %spec.module_id, "supervised module disabled");
8018        Ok(true)
8019    }
8020}
8021
8022#[allow(clippy::too_many_arguments)]
8023async fn on_child_exit(
8024    spec: &ModuleSpec,
8025    policy: RestartPolicy,
8026    registry: &Registry,
8027    snapshot: &SharedSnapshot,
8028    terminal_ring: &Arc<Mutex<TerminalRing>>,
8029    spawn_events: &SpawnEventFeed,
8030    roster: &ChildRoster,
8031    exit_report: ExitReport,
8032    daemon_initiated: bool,
8033) -> NextAction {
8034    // Once the daemon has begun shutting down, no exit is a crash to recover
8035    // from: the module is exiting because the daemon is going away (EOF on its
8036    // connection, or a service manager signalling the whole cgroup). Record it
8037    // as such and never schedule a respawn, which would only start a process
8038    // for the shutdown to end again.
8039    if roster.is_closed() {
8040        return on_child_exit_during_daemon_shutdown(
8041            spec,
8042            registry,
8043            snapshot,
8044            terminal_ring,
8045            spawn_events,
8046            exit_report,
8047        )
8048        .await;
8049    }
8050    // Deliberate stops the supervisor asks for (operator stop, disable,
8051    // restart, reload, swap, a health restart, a drain that runs out of budget)
8052    // takes the child out of the supervise loop and reaps it in
8053    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8054    // that reaches this point is either unrequested or a first-HELLO timeout,
8055    // which is deliberately classified as a crash even after a clean TERM exit.
8056    //
8057    // For a subc-wire module a clean exit is still a stop: those modules are
8058    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8059    // crash, and exiting 0 is a deliberate choice the module made. A
8060    // `protocol: "none"` module is a stock program we cannot change, and many
8061    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8062    // stop would leave the module down for good after any stray signal, so it
8063    // goes through the crash path instead: it spends restart budget, respawns
8064    // with the crash backoff, and ends `failed` when the budget runs out.
8065    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8066        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8067    match exit_report.kind {
8068        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8069            info!(
8070                module_id = %spec.module_id,
8071                exit_code = ?exit_report.code,
8072                exit_signal = ?exit_report.signal,
8073                "supervised module exited cleanly"
8074            );
8075            if let Err(err) = publish_after_terminal_record(
8076                &spec.module_id,
8077                || {
8078                    record_terminal(
8079                        &spec.module_id,
8080                        terminal_ring,
8081                        spawn_events,
8082                        &exit_report,
8083                        TerminalDisposition::Stopped,
8084                    )
8085                },
8086                || {
8087                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8088                        state.state = ModuleState::Stopped;
8089                        clear_current_process_facts(state);
8090                        state.last_exit = Some(exit_report.clone());
8091                    })
8092                },
8093            ) {
8094                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8095            }
8096            let registration_released = match wait_for_registration_release(
8097                registry,
8098                &spec.module_id,
8099                REGISTRY_RELEASE_TIMEOUT,
8100            )
8101            .await
8102            {
8103                Ok(()) => true,
8104                Err(err) => {
8105                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8106                    false
8107                }
8108            };
8109            NextAction::Stop {
8110                registration_released,
8111            }
8112        }
8113        ExitKind::Clean | ExitKind::Crash => {
8114            if unrequested_clean_exit_of_protocol_none {
8115                warn!(
8116                    module_id = %spec.module_id,
8117                    exit_code = ?exit_report.code,
8118                    exit_signal = ?exit_report.signal,
8119                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8120                );
8121            } else {
8122                warn!(
8123                    module_id = %spec.module_id,
8124                    exit_code = ?exit_report.code,
8125                    exit_signal = ?exit_report.signal,
8126                    "supervised module exited abnormally (crash)"
8127                );
8128            }
8129            let mut restart_schedule = None;
8130            let mut disposition = TerminalDisposition::Disabled;
8131            let mut final_state = ModuleState::Disabled;
8132            // Set only when the budget is what stopped the module, so the
8133            // terminal record says which limit was hit rather than leaving
8134            // `failed` to be read as "crashed once, badly".
8135            let mut disposition_detail = lock_snapshot(snapshot)
8136                .ok()
8137                .and_then(|mut state| state.spawn_failure.take());
8138            let now = Instant::now();
8139            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8140                if state.enabled {
8141                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8142                        // Reserving the last slot exhausts the budget for the
8143                        // replacement, not for this exit. Publish its scheduled
8144                        // recovery atomically with the reservation so connection
8145                        // cleanup sees it while terminal recording still holds
8146                        // the visible state at Running.
8147                        state.respawn_pending = true;
8148                        final_state = ModuleState::Restarting;
8149                        restart_schedule = Some(schedule);
8150                        disposition = TerminalDisposition::Restarting;
8151                    } else {
8152                        final_state = ModuleState::Failed;
8153                        disposition = TerminalDisposition::Failed;
8154                        let budget = policy.budget_exhausted_detail();
8155                        disposition_detail =
8156                            Some(disposition_detail.take().map_or_else(
8157                                || budget.clone(),
8158                                |cause| format!("{cause}; {budget}"),
8159                            ));
8160                    }
8161                } else {
8162                    disposition = TerminalDisposition::Disabled;
8163                }
8164            }) {
8165                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8166                return NextAction::Stop {
8167                    registration_released: false,
8168                };
8169            }
8170            if disposition == TerminalDisposition::Failed {
8171                // The window is in the message, not only in the fields: this line
8172                // is read in a scrollback where a bare `max_restarts=3` reads as a
8173                // lifetime cap and sends the operator looking for three crashes
8174                // that never happened together.
8175                error!(
8176                    module_id = %spec.module_id,
8177                    max_restarts = policy.max_restarts,
8178                    window_secs = policy.window.as_secs(),
8179                    "module stopped: {}",
8180                    policy.budget_exhausted_detail()
8181                );
8182            }
8183            if let Err(err) = publish_after_terminal_record(
8184                &spec.module_id,
8185                || {
8186                    record_terminal_with_detail(
8187                        &spec.module_id,
8188                        terminal_ring,
8189                        spawn_events,
8190                        &exit_report,
8191                        disposition,
8192                        disposition_detail,
8193                    )
8194                },
8195                || {
8196                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8197                        state.state = final_state;
8198                        clear_current_process_facts(state);
8199                        if final_state == ModuleState::Failed
8200                            && daemon_initiated
8201                            && !roster.is_closed()
8202                        {
8203                            state.failed_retry.schedule(&spec.module_id);
8204                        }
8205                        state.last_exit = Some(exit_report.clone());
8206                    })
8207                },
8208            ) {
8209                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8210            }
8211
8212            if let Some(schedule) = restart_schedule {
8213                NextAction::Restart {
8214                    schedule: Some(schedule),
8215                }
8216            } else {
8217                let registration_released = match wait_for_registration_release(
8218                    registry,
8219                    &spec.module_id,
8220                    REGISTRY_RELEASE_TIMEOUT,
8221                )
8222                .await
8223                {
8224                    Ok(()) => true,
8225                    Err(err) => {
8226                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8227                        false
8228                    }
8229                };
8230                NextAction::Stop {
8231                    registration_released,
8232                }
8233            }
8234        }
8235        ExitKind::DeliberateSeverance => {
8236            warn!(
8237                module_id = %spec.module_id,
8238                exit_code = ?exit_report.code,
8239                exit_signal = ?exit_report.signal,
8240                "supervised module exited after deliberate connection severance"
8241            );
8242            let should_restart = match lock_snapshot(snapshot) {
8243                Ok(state) => state.enabled,
8244                Err(err) => {
8245                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8246                    return NextAction::Stop {
8247                        registration_released: false,
8248                    };
8249                }
8250            };
8251            let final_state = if should_restart {
8252                ModuleState::Restarting
8253            } else {
8254                ModuleState::Disabled
8255            };
8256            if let Err(err) = publish_after_terminal_record(
8257                &spec.module_id,
8258                || {
8259                    record_terminal(
8260                        &spec.module_id,
8261                        terminal_ring,
8262                        spawn_events,
8263                        &exit_report,
8264                        terminal_disposition(final_state),
8265                    )
8266                },
8267                || {
8268                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8269                        state.state = final_state;
8270                        clear_current_process_facts(state);
8271                        state.last_exit = Some(exit_report.clone());
8272                        state.lifetime_restarts += 1;
8273                    })
8274                },
8275            ) {
8276                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8277                return NextAction::Stop {
8278                    registration_released: false,
8279                };
8280            }
8281
8282            if should_restart {
8283                NextAction::Restart { schedule: None }
8284            } else {
8285                let registration_released = match wait_for_registration_release(
8286                    registry,
8287                    &spec.module_id,
8288                    REGISTRY_RELEASE_TIMEOUT,
8289                )
8290                .await
8291                {
8292                    Ok(()) => true,
8293                    Err(err) => {
8294                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8295                        false
8296                    }
8297                };
8298                NextAction::Stop {
8299                    registration_released,
8300                }
8301            }
8302        }
8303    }
8304}
8305
8306async fn on_child_exit_during_daemon_shutdown(
8307    spec: &ModuleSpec,
8308    registry: &Registry,
8309    snapshot: &SharedSnapshot,
8310    terminal_ring: &Arc<Mutex<TerminalRing>>,
8311    spawn_events: &SpawnEventFeed,
8312    exit_report: ExitReport,
8313) -> NextAction {
8314    info!(
8315        module_id = %spec.module_id,
8316        exit_code = ?exit_report.code,
8317        exit_signal = ?exit_report.signal,
8318        exit_kind = ?exit_report.kind,
8319        "supervised module exited during daemon shutdown; not restarting it"
8320    );
8321    if let Err(err) = publish_after_terminal_record(
8322        &spec.module_id,
8323        || {
8324            record_terminal(
8325                &spec.module_id,
8326                terminal_ring,
8327                spawn_events,
8328                &exit_report,
8329                TerminalDisposition::DaemonShutdown,
8330            )
8331        },
8332        || {
8333            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8334                state.state = ModuleState::Stopped;
8335                clear_current_process_facts(state);
8336                state.last_exit = Some(exit_report.clone());
8337            })
8338        },
8339    ) {
8340        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8341    }
8342    let registration_released =
8343        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8344            .await
8345            .is_ok();
8346    NextAction::Stop {
8347        registration_released,
8348    }
8349}
8350
8351// Keep the status lock out of terminal I/O. Readers may observe the previous
8352// state while recording is in progress, but never a completed exit without its
8353// history. Journal errors are logged by the journal and retained in the ring;
8354// a recording panic still publishes the final state before resuming the unwind.
8355fn publish_after_terminal_record<T>(
8356    module_id: &str,
8357    record: impl FnOnce(),
8358    publish: impl FnOnce() -> T,
8359) -> T {
8360    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8361    let published = publish();
8362    if let Err(panic) = recorded {
8363        error!(
8364            module_id,
8365            "terminal recording panicked; published final state before resuming unwind"
8366        );
8367        std::panic::resume_unwind(panic);
8368    }
8369    published
8370}
8371
8372fn fail_after_wait_error(
8373    module_id: &str,
8374    snapshot: &SharedSnapshot,
8375    terminal_ring: &Arc<Mutex<TerminalRing>>,
8376    spawn_events: &SpawnEventFeed,
8377    detail: String,
8378    failure: &FailureContext,
8379) {
8380    let report = wait_error_exit_report();
8381    publish_after_terminal_record(
8382        module_id,
8383        || {
8384            record_terminal_with_detail(
8385                module_id,
8386                terminal_ring,
8387                spawn_events,
8388                &report,
8389                TerminalDisposition::Failed,
8390                Some(detail),
8391            )
8392        },
8393        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8394    );
8395}
8396
8397#[cfg(test)]
8398fn record_wait_error_terminal(
8399    module_id: &str,
8400    terminal_ring: &Arc<Mutex<TerminalRing>>,
8401    spawn_events: &SpawnEventFeed,
8402) {
8403    record_terminal(
8404        module_id,
8405        terminal_ring,
8406        spawn_events,
8407        &wait_error_exit_report(),
8408        TerminalDisposition::Failed,
8409    );
8410}
8411
8412fn record_terminal(
8413    module_id: &str,
8414    terminal_ring: &Arc<Mutex<TerminalRing>>,
8415    spawn_events: &SpawnEventFeed,
8416    exit_report: &ExitReport,
8417    disposition: TerminalDisposition,
8418) {
8419    record_terminal_with_detail(
8420        module_id,
8421        terminal_ring,
8422        spawn_events,
8423        exit_report,
8424        disposition,
8425        None,
8426    );
8427}
8428
8429/// The ring lock is held only to capture the read (see
8430/// `TerminalJournal::capture_read`), so this module's exits keep recording
8431/// while the journal files are read. Blocking: it reads files.
8432fn durable_terminal_history_of(
8433    terminal_ring: &Mutex<TerminalRing>,
8434    module_id: &str,
8435) -> subc_control::TerminalHistory {
8436    let read = terminal_ring
8437        .lock()
8438        .unwrap_or_else(|p| p.into_inner())
8439        .capture_durable_history();
8440    read.read(module_id)
8441}
8442
8443fn record_terminal_with_detail(
8444    module_id: &str,
8445    terminal_ring: &Arc<Mutex<TerminalRing>>,
8446    spawn_events: &SpawnEventFeed,
8447    exit_report: &ExitReport,
8448    disposition: TerminalDisposition,
8449    disposition_detail: Option<String>,
8450) {
8451    #[cfg(test)]
8452    terminal_history_tests::before_terminal_record(spawn_events);
8453    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8454    let record = TerminalRecord {
8455        exit_code: exit_report.code,
8456        exit_signal: exit_report.signal,
8457        at_ms: exit_report.at_ms,
8458        disposition,
8459        exit_kind: exit_report.kind.into(),
8460        disposition_detail,
8461    };
8462    terminal_ring
8463        .lock()
8464        .unwrap_or_else(|poisoned| poisoned.into_inner())
8465        .record_exit(module_id, record);
8466}
8467
8468fn untrack_if_registration_released(
8469    process_liveness: &SupervisorProcessLiveness,
8470    registry: &Registry,
8471    module_id: &str,
8472    snapshot: &SharedSnapshot,
8473) {
8474    match registry.get_module(module_id) {
8475        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8476        Ok(Some(_)) => {}
8477        Err(err) => {
8478            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8479        }
8480    }
8481}
8482
8483/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8484/// then apply the module's configured entries minus daemon-private capture keys.
8485///
8486/// Separated from `spawn_child` only so it can be asserted without spawning a
8487/// process — a duplicate of this logic in a test would pass while the real one
8488/// drifted, which is the defect class this function exists to avoid.
8489/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8490/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8491/// either and the argument would stop a stock binary from starting at all.
8492/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8493///
8494/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8495/// through [`apply_wire_spawn_args_for_role`].
8496#[cfg(test)]
8497fn apply_wire_spawn_args(
8498    command: &mut Command,
8499    spec: &ModuleSpec,
8500    connection_file_path: Option<&std::path::Path>,
8501    handle: Option<&SupervisorHandle>,
8502) -> Result<Option<NonceHandoff>, SuperviseError> {
8503    apply_wire_spawn_args_for_role(
8504        command,
8505        spec,
8506        connection_file_path,
8507        handle,
8508        SpawnRole::Plain,
8509    )
8510}
8511
8512/// The read end of a spawn's launch-nonce pipe, prepared by
8513/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8514/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8515/// server before spawn and authenticates its client after the child exists.
8516#[cfg(unix)]
8517type NonceHandoff = subc_os::LaunchNonceHandoff;
8518#[cfg(windows)]
8519type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8520#[cfg(not(any(unix, windows)))]
8521type NonceHandoff = std::convert::Infallible;
8522
8523/// Create the launch nonce that a new module process must present when it
8524/// registers or opens a consumer connection, including a replacement process
8525/// started alongside a still-serving module.
8526///
8527/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8528/// a separate candidate token so the still-serving incumbent and its consumers
8529/// keep their nonce. Both records are installed before the process exists, so
8530/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8531///
8532/// On Unix the nonce is delivered only through a pipe. It is written into
8533/// a pipe whose read end the child gets as descriptor 3, named by
8534/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8535/// process of the same user cannot read it with `ps eww`. That handoff is
8536/// returned rather than installed here, because installing it replaces
8537/// whatever the child has at descriptor 3 and so must be the last pre-exec
8538/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8539/// the Linux callback that writes the process into its cgroup. Only after that
8540/// does `handoff.install_last` register the descriptor-3 replacement, so it
8541/// cannot close a descriptor the cgroup callback still needs.
8542/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8543fn apply_wire_spawn_args_for_role(
8544    command: &mut Command,
8545    spec: &ModuleSpec,
8546    connection_file_path: Option<&std::path::Path>,
8547    handle: Option<&SupervisorHandle>,
8548    role: SpawnRole,
8549) -> Result<Option<NonceHandoff>, SuperviseError> {
8550    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8551    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8552    // included: a daemon started from a module's process tree inherits it,
8553    // and passing it on would point the child at a descriptor it does not
8554    // have.
8555    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8556    #[cfg(windows)]
8557    {
8558        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8559        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8560    }
8561    // Remove inherited or configured copies too: withholding must mean absent.
8562    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8563    if spec.protocol == ModuleProtocol::None {
8564        return Ok(None);
8565    }
8566    if let Some(connection_file_path) = connection_file_path {
8567        command.arg(SUBC_ARG).arg(connection_file_path);
8568    }
8569
8570    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8571    // route.open attestation. Reserved modules additionally use the same nonce
8572    // for HELLO id-squatting protection. A respawn rotates both records.
8573    let nonce = generate_launch_nonce()?;
8574    if let Some(handle) = handle {
8575        match role {
8576            SpawnRole::Plain => {
8577                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8578                if spec.reserved {
8579                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8580                }
8581            }
8582            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8583        }
8584    }
8585    #[cfg(unix)]
8586    let handoff = {
8587        let handoff =
8588            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8589                program: spec.program.clone(),
8590                source,
8591                cgroup_path: None,
8592            })?;
8593        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8594        Some(handoff)
8595    };
8596    #[cfg(windows)]
8597    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8598    #[cfg(not(any(unix, windows)))]
8599    let handoff = None;
8600    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8601    // until the daemon's per-spawn delivery records show that every running
8602    // Windows module has read its secret from the named pipe.
8603    #[cfg(not(unix))]
8604    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8605    Ok(handoff)
8606}
8607
8608fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8609    command.env_remove(CK_LOG_ENV);
8610    // The spawn role is the supervisor's to set, and only on a swap candidate
8611    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8612    // it, is what makes it absent on a plain spawn: the daemon's own
8613    // environment could carry it, and so could a spec built outside daemon
8614    // config (config refuses it as an `env` key). A module reading it on a
8615    // plain restart would pick the long swap budget and leave callers waiting.
8616    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8617    for (key, value) in &spec.env {
8618        // cortexkit-log currently exposes retention only as a Rust struct, not
8619        // environment names. These values are daemon-private sink metadata and
8620        // must never become a public child-process contract by being inherited.
8621        if matches!(
8622            key.as_str(),
8623            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8624        ) || key == SUBC_SPAWN_ROLE_ENV
8625        {
8626            continue;
8627        }
8628        command.env(key, value);
8629    }
8630}
8631
8632#[cfg(windows)]
8633fn prepare_windows_nonce_pipe(
8634    command: &mut Command,
8635    spec: &ModuleSpec,
8636    nonce: &str,
8637) -> Option<NonceHandoff> {
8638    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8639        Ok(handoff) => {
8640            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8641            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8642            // still register until every module reads the pipe. A reader without
8643            // this variable set to env returns the pipe error instead of using
8644            // an inherited or configured environment copy.
8645            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8646            Some(handoff)
8647        }
8648        Err(error) => {
8649            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8650            None
8651        }
8652    }
8653}
8654
8655#[cfg(windows)]
8656fn start_windows_nonce_pipe(
8657    handoff: Option<NonceHandoff>,
8658    pid: u32,
8659    spec: &ModuleSpec,
8660) -> Option<subc_os::LaunchNoncePipeDelivery> {
8661    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8662    // same maximum wait used for registration during a reload. Older modules
8663    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8664    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8665        Ok(delivery) => Some(delivery),
8666        Err(error) => {
8667            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8668            None
8669        }
8670    })
8671}
8672
8673fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8674    #[cfg(windows)]
8675    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8676        return Some(
8677            if snapshot
8678                .pipe_consumed
8679                .as_ref()
8680                .is_some_and(|consumed| consumed.get().is_some())
8681            {
8682                "pipe"
8683            } else {
8684                "env"
8685            }
8686            .to_owned(),
8687        );
8688    }
8689    let _ = snapshot;
8690    None
8691}
8692
8693/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8694/// of a blue/green swap.
8695#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8696enum SpawnRole {
8697    Plain,
8698    SwapCandidate,
8699}
8700
8701/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8702/// `apply_child_env` has already removed the variable for every spawn.
8703fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8704    if role == SpawnRole::SwapCandidate {
8705        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8706    }
8707}
8708
8709fn previous_instance_error(
8710    spec: &ModuleSpec,
8711    pid: u32,
8712    reason: crate::live_children::RetainedReason,
8713) -> SuperviseError {
8714    SuperviseError::Spawn {
8715        program: spec.program.clone(),
8716        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8717        cgroup_path: None,
8718    }
8719}
8720
8721fn spawn_child(
8722    spec: &ModuleSpec,
8723    connection_file_path: Option<&std::path::Path>,
8724    handle: Option<&SupervisorHandle>,
8725    ring: &Arc<Mutex<StderrRing>>,
8726    capture_logs_dir: Option<&std::path::Path>,
8727    roster: &ChildRoster,
8728    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8729) -> Result<SupervisedChild, SuperviseError> {
8730    spawn_child_in_slot(
8731        spec,
8732        connection_file_path,
8733        handle,
8734        ring,
8735        capture_logs_dir,
8736        roster,
8737        #[cfg(target_os = "linux")]
8738        cgroup_placement,
8739        SpawnRole::Plain,
8740        false,
8741    )
8742}
8743
8744/// Spawn one process of `spec` into a slot.
8745///
8746/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8747/// A swap candidate needs a different cgroup from the process it is replacing,
8748/// which is still alive: in the same cgroup the two would be one kill domain,
8749/// and killing a failed candidate could take the incumbent with it.
8750///
8751/// The stderr capture file is `<module_id>.stderr.log` for every process of
8752/// the module, whichever slot it is in, because that is the one file
8753/// `ck module logs` reads. During a swap's overlap both processes append to it;
8754/// the daemon writes whole lines, so the two interleave by line, which is also
8755/// the merged view an operator wants while a swap runs.
8756#[allow(clippy::too_many_arguments)]
8757fn spawn_child_in_slot(
8758    spec: &ModuleSpec,
8759    connection_file_path: Option<&std::path::Path>,
8760    handle: Option<&SupervisorHandle>,
8761    ring: &Arc<Mutex<StderrRing>>,
8762    capture_logs_dir: Option<&std::path::Path>,
8763    roster: &ChildRoster,
8764    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8765    role: SpawnRole,
8766    alternate_slot: bool,
8767) -> Result<SupervisedChild, SuperviseError> {
8768    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8769        return Err(previous_instance_error(spec, pid, reason));
8770    }
8771    if roster.is_closed() {
8772        return Err(SuperviseError::Spawn {
8773            program: spec.program.clone(),
8774            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8775            cgroup_path: None,
8776        });
8777    }
8778    #[cfg(target_os = "linux")]
8779    let cgroup_name = {
8780        // Slot names alone are not kill domains: a retired incumbent may still
8781        // be draining when a later enable/restart spawns into the same slot.
8782        // Decimal entropy keeps the suffix unambiguous; Placement performs
8783        // the module-id escaping and constructs the filesystem path.
8784        if cgroup_placement.is_none() {
8785            swap::cgroup_name(&spec.module_id, alternate_slot)
8786        } else {
8787            let nonce = generate_launch_nonce()?;
8788            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8789            // Leave room for byte escaping and the suffix under NAME_MAX. The
8790            // label is only for humans; the nonce identifies the kill domain.
8791            let mut end = spec.module_id.len().min(64);
8792            while !spec.module_id.is_char_boundary(end) {
8793                end -= 1;
8794            }
8795            format!(
8796                "{}_{suffix}",
8797                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8798            )
8799        }
8800    };
8801    #[cfg(not(target_os = "linux"))]
8802    let _ = alternate_slot;
8803    #[cfg(target_os = "macos")]
8804    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8805    #[cfg(windows)]
8806    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8807        program: spec.program.clone(),
8808        source,
8809        cgroup_path: None,
8810    })?;
8811    #[cfg(windows)]
8812    let mut command = Command::new(image_capture.path());
8813    #[cfg(not(any(target_os = "macos", windows)))]
8814    let mut command = Command::new(&spec.program);
8815    command.args(&spec.args);
8816    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8817    // that is the whole of the intent, so remove that one key rather than the
8818    // environment.
8819    //
8820    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8821    // and took the POSIX environment with it. Modules spawned that way had no
8822    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8823    // logging:
8824    //
8825    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8826    //     both unset it fell back to the temp dir alone and `ck` could not find
8827    //     a daemon running on the same machine from inside any module's process
8828    //     tree — reporting a path the file has never lived at, which reads as
8829    //     "the daemon did not write its file".
8830    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8831    //     the RELATIVE `.local/share`, so a module deriving its own store path
8832    //     resolved it against its own CWD. That is the store-fragmentation
8833    //     defect the daemon already refuses in config (`parse_doc` rejects a
8834    //     relative `storage.data_home`) arriving by derivation instead.
8835    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8836    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8837    //     quietly rather than erroring.
8838    //
8839    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8840    // offered one candidate under /tmp while the file sat in /run/user/1000.
8841    //
8842    // A configured module is unaffected either way: `module_spec()` puts the
8843    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8844    // wins over anything ambient.
8845    apply_child_env(&mut command, spec);
8846    apply_spawn_role(&mut command, role);
8847    let nonce_handoff =
8848        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8849
8850    #[cfg(target_os = "linux")]
8851    let cgroup_path = cgroup_placement
8852        .map(|placement| placement.module_path(&cgroup_name))
8853        .transpose()
8854        .map_err(|source| SuperviseError::Cgroup {
8855            module_id: spec.module_id.clone(),
8856            source,
8857        })?;
8858    #[cfg(not(target_os = "linux"))]
8859    let cgroup_path: Option<PathBuf> = None;
8860    #[cfg(target_os = "linux")]
8861    if let Some(path) = &cgroup_path {
8862        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8863            if let Some(placement) = cgroup_placement {
8864                remove_module_cgroup(placement, &cgroup_name);
8865            }
8866            return Err(error);
8867        }
8868    }
8869
8870    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8871        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8872        match ChildOutputSink::open(&path, capture_retention(spec)) {
8873            Ok(sink) => sink,
8874            Err(error) => {
8875                warn!(
8876                    module_id = %spec.module_id,
8877                    path = %path.display(),
8878                    error = %error,
8879                    "could not open child output capture file; forwarding to stderr"
8880                );
8881                ChildOutputSink::Stderr
8882            }
8883        }
8884    } else {
8885        ChildOutputSink::Stderr
8886    };
8887
8888    command.stdout(Stdio::piped());
8889    command.stderr(Stdio::piped());
8890    command.kill_on_drop(true);
8891    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8892    // before exec). In the daemon's group, a service manager that kills the
8893    // job's process group when the daemon exits (launchd's default) killed
8894    // every module at the same moment its control connection closed, so no
8895    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8896    // module is reached only by the daemon: the EOF it sees when its
8897    // connection closes, and the bounded stop in `child_roster` for anything
8898    // still running after that. On Linux this composes with the cgroup
8899    // placement above: that is a pre_exec write to cgroup.procs, std performs
8900    // setpgid in the child before running pre_exec callbacks, and the two
8901    // change independent process attributes.
8902    //
8903    // stdin is /dev/null because a process outside the terminal's foreground
8904    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8905    // by hand would otherwise hand down. Under a service manager stdin is
8906    // already /dev/null.
8907    #[cfg(unix)]
8908    command.process_group(0);
8909    command.stdin(Stdio::null());
8910    // The LAST pre-exec step, after the cgroup placement above: installing the
8911    // nonce at descriptor 3 replaces whatever the child had there, which could
8912    // be the descriptor an earlier step writes through.
8913    #[cfg(unix)]
8914    if let Some(handoff) = nonce_handoff {
8915        handoff.install_last(command.as_std_mut());
8916    }
8917    #[cfg(not(any(unix, windows)))]
8918    let _ = nonce_handoff;
8919
8920    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8921    // cannot run a single instruction -- and therefore cannot spawn a
8922    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8923    // other two steps and why the window matters.
8924    #[cfg(windows)]
8925    {
8926        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8927            program: spec.program.clone(),
8928            source,
8929            cgroup_path: None,
8930        })?;
8931        subc_jobobject::suspend_on_create_async(&mut command);
8932    }
8933    // Timestamp before the spawn call: on Unix the child can serve before that
8934    // call even returns. This conservative bound never labels an older child
8935    // as a replacement for a request received during process creation.
8936    let spawned_at = Instant::now();
8937    let mut child = match command.spawn() {
8938        Ok(child) => child,
8939        Err(source) => {
8940            #[cfg(target_os = "linux")]
8941            if let Some(placement) = cgroup_placement {
8942                remove_module_cgroup(placement, &cgroup_name);
8943            }
8944            return Err(SuperviseError::Spawn {
8945                program: spec.program.clone(),
8946                source,
8947                cgroup_path,
8948            });
8949        }
8950    };
8951    #[cfg(test)]
8952    spawn_record_tests::after_create(&child, spec);
8953    // The parent must close its writer now: the acknowledgement pipe reports EOF
8954    // only when every writer is gone, and the child's copy closes when the
8955    // trampoline replaces itself with the module. Command holds only an integer
8956    // in its pre_exec callback, not another writer.
8957    #[cfg(target_os = "macos")]
8958    drop(exec_ack);
8959
8960    // Bind while the child is still suspended, before job assignment resumes it.
8961    // Keeping the executable handle open across CreateProcessW denies rename
8962    // and deletion, so the file cannot be exchanged during spawn. Bind that
8963    // file ID to the process handle returned by spawn, never a reopened PID.
8964    #[cfg(windows)]
8965    let windows_image = match image_capture.bind_tokio(&child) {
8966        Ok(image) => Arc::new(image),
8967        Err(source) => {
8968            kill_and_reap_rejected_child(&mut child);
8969            return Err(SuperviseError::Spawn {
8970                program: spec.program.clone(),
8971                source,
8972                cgroup_path: None,
8973            });
8974        }
8975    };
8976    // Assign the suspended child to its module job; resume only after its
8977    // process identity has been durably recorded for orphan recovery.
8978    #[cfg(windows)]
8979    let job = contain_spawned_child(&mut child, spec)?;
8980    let spawned_at_ms = unix_ms_now();
8981    #[cfg(not(windows))]
8982    let spawned_from = spec.program.clone();
8983    #[cfg(windows)]
8984    let spawned_from = windows_image.path().to_path_buf();
8985    #[cfg(not(windows))]
8986    let spawned_file_identity = spawned_file_identity(&spawned_from);
8987    #[cfg(windows)]
8988    let spawned_file_identity = Some(SpawnedFileIdentity {
8989        device: 0,
8990        inode: 0,
8991        windows_image: Some(Arc::clone(&windows_image)),
8992    });
8993    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8994        program: spec.program.clone(),
8995        source: io::Error::other("spawned child exposed no live pid"),
8996        cgroup_path: cgroup_path.clone(),
8997    })?;
8998    #[cfg(windows)]
8999    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9000    #[cfg(not(windows))]
9001    let process_start_time = crate::provenance::process_start_time(pid);
9002    #[cfg(windows)]
9003    let process_start_time = Some(windows_image.start_time());
9004    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9005    #[cfg(all(test, target_os = "macos"))]
9006    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9007    // Record the committed kernel image, not the configured pathname: PATH
9008    // lookup and shebang interpretation can select a different executable.
9009    // Linux's posix_spawn can return before that image is installed, so the
9010    // observer also waits for the kernel's first-exec flag to clear.
9011    #[cfg(not(windows))]
9012    let recorded_image = observe_spawned_image(pid);
9013    // spawn() confirms only the first exec, into the trampoline. Never persist
9014    // the trampoline image; the asynchronous acknowledgement publishes the
9015    // module image once the trampoline has replaced itself with the module.
9016    #[cfg(target_os = "macos")]
9017    let recorded_image = if privacy_exec.is_some() {
9018        None
9019    } else {
9020        recorded_image
9021    };
9022    #[cfg(target_os = "linux")]
9023    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9024    #[cfg(not(target_os = "linux"))]
9025    let recorded_cgroup_name = None;
9026    let roster_guard = roster.admit(
9027        spec.module_id.clone(),
9028        pid,
9029        spec.protocol,
9030        process_start_time,
9031        crate::child_roster::RecordedIdentity {
9032            #[cfg(not(windows))]
9033            start_time: recorded_image.map(|image| image.start_time),
9034            #[cfg(windows)]
9035            start_time: process_start_time,
9036            #[cfg(not(windows))]
9037            executable: recorded_image
9038                .and_then(|image| image.executable)
9039                .map(crate::live_children::ExecutableIdentity::from),
9040            #[cfg(windows)]
9041            executable: Some(crate::live_children::ExecutableIdentity::from(
9042                windows_image.identity(),
9043            )),
9044            cgroup_name: recorded_cgroup_name,
9045            #[cfg(target_os = "linux")]
9046            cgroup_placement: cgroup_placement.cloned(),
9047        },
9048    );
9049    let roster_guard = match roster_guard {
9050        Ok(guard) => guard,
9051        Err(failure) => {
9052            return failure.with_cleanup(|source| {
9053                #[cfg(windows)]
9054                if let Some(job) = &job {
9055                    let _ = job.terminate();
9056                }
9057                #[cfg(target_os = "linux")]
9058                kill_module_cgroup(cgroup_placement, &cgroup_name);
9059                kill_and_reap_rejected_child(&mut child);
9060                #[cfg(target_os = "linux")]
9061                if let Some(placement) = cgroup_placement {
9062                    remove_module_cgroup(placement, &cgroup_name);
9063                }
9064                Err(SuperviseError::Spawn {
9065                    program: spec.program.clone(),
9066                    source: io::Error::other(format!(
9067                        "could not durably record module child: {source}"
9068                    )),
9069                    cgroup_path,
9070                })
9071            });
9072        }
9073    };
9074    // The pre-spawn check can pass just before shutdown begins. Ownership is
9075    // published at the start of admission, even while a record write is queued.
9076    // If shutdown already saw an empty roster before that publication, this
9077    // check sees the roster closed and ends the child before starting a module.
9078    if roster.is_closed() {
9079        // This child was never admitted, so there is no module protocol shutdown to wait for.
9080        #[cfg(target_os = "linux")]
9081        kill_module_cgroup(cgroup_placement, &cgroup_name);
9082        if let Err(error) = child.start_kill() {
9083            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9084        }
9085        #[cfg(target_os = "linux")]
9086        if let Some(placement) = cgroup_placement {
9087            // This spawn was never admitted, so shutdown has no roster entry
9088            // to await. Do not detach its cleanup: the runtime could exit
9089            // before that task reaps the rejected child and removes its group.
9090            while matches!(child.try_wait(), Ok(None)) {
9091                std::thread::yield_now();
9092            }
9093            if matches!(
9094                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9095                subc_cgroup::KillOutcome::Killed
9096            ) {
9097                if let Ok(path) = placement.module_path(&cgroup_name) {
9098                    while std::fs::read_to_string(path.join("cgroup.events"))
9099                        .ok()
9100                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9101                    {
9102                        std::thread::yield_now();
9103                    }
9104                }
9105            }
9106            remove_module_cgroup(placement, &cgroup_name);
9107        }
9108        drop(roster_guard);
9109        return Err(SuperviseError::Spawn {
9110            program: spec.program.clone(),
9111            source: io::Error::other(
9112                "the daemon began shutting down while this process was starting; ended it",
9113            ),
9114            cgroup_path,
9115        });
9116    }
9117
9118    #[cfg(windows)]
9119    if let Err(error) = resume_suspended_child(pid, spec) {
9120        kill_and_reap_rejected_child(&mut child);
9121        return Err(error);
9122    }
9123
9124    let stdout_pump = match child.stdout.take() {
9125        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9126        None => {
9127            warn!(
9128                module_id = %spec.module_id,
9129                "spawned child exposed no stdout pipe; file capture will be incomplete"
9130            );
9131            None
9132        }
9133    };
9134    let stderr_pump = match child.stderr.take() {
9135        Some(stderr) => {
9136            let generation = ring
9137                .lock()
9138                .unwrap_or_else(|poisoned| poisoned.into_inner())
9139                .begin_process();
9140            Some(StderrPump {
9141                task: tokio::spawn(pump_stderr_to(
9142                    stderr,
9143                    Arc::clone(ring),
9144                    generation,
9145                    output_sink,
9146                )),
9147                generation,
9148            })
9149        }
9150        None => {
9151            // Spawning succeeded but the pipe did not materialise. Recording it as
9152            // uncaptured keeps the tail honest: the alternative is an empty tail
9153            // that reads as a module which printed nothing.
9154            ring.lock()
9155                .unwrap_or_else(|poisoned| poisoned.into_inner())
9156                .mark_not_captured("stderr pipe was not available on spawn");
9157            warn!(
9158                module_id = %spec.module_id,
9159                "spawned child exposed no stderr pipe; tail will be unavailable"
9160            );
9161            None
9162        }
9163    };
9164
9165    let exec = Arc::new(crate::activity::ExecOutcome::new());
9166    #[cfg(target_os = "macos")]
9167    let pending_exec = privacy_exec.is_some();
9168    #[cfg(not(target_os = "macos"))]
9169    let pending_exec = false;
9170    if !pending_exec {
9171        exec.finish(crate::activity::ExecState::CommittedExec);
9172    }
9173    Ok(SupervisedChild {
9174        child,
9175        exec: crate::activity::ExecOwner(exec),
9176        #[cfg(windows)]
9177        nonce_delivery,
9178        protocol: spec.protocol,
9179        #[cfg(target_os = "linux")]
9180        module_id: cgroup_name,
9181        #[cfg(target_os = "linux")]
9182        cgroup_placement: cgroup_placement.cloned(),
9183        #[cfg(windows)]
9184        job,
9185        stdout_pump,
9186        stderr_pump,
9187        stderr_ring: Arc::clone(ring),
9188        spawned_at,
9189        registration_deadline: None,
9190        registration_watch: None,
9191        spawned_at_ms,
9192        spawned_from,
9193        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9194        spawned_file_identity,
9195        process_start_time,
9196        process_identity,
9197        pid,
9198        roster_guard: Some(roster_guard),
9199        #[cfg(target_os = "macos")]
9200        privacy_exec,
9201        #[cfg(target_os = "macos")]
9202        report_ready: Arc::new(OnceLock::new()),
9203        spawn_failure: None,
9204    })
9205}
9206
9207#[cfg(test)]
9208mod spawn_record_tests {
9209    use super::*;
9210
9211    thread_local! {
9212        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9213    }
9214
9215    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9216        LAST_PID.set(child.id());
9217        #[cfg(windows)]
9218        if let Some((_, address)) = _spec
9219            .env
9220            .iter()
9221            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9222        {
9223            use std::io::{Read, Write};
9224            let mut stream = std::net::TcpStream::connect(address).unwrap();
9225            stream
9226                .write_all(&child.id().unwrap().to_ne_bytes())
9227                .unwrap();
9228            let mut release = [0];
9229            stream.read_exact(&mut release).unwrap();
9230        }
9231    }
9232
9233    #[cfg(windows)]
9234    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9235        if spec
9236            .env
9237            .iter()
9238            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9239        {
9240            Err(io::Error::other("injected module job assignment failure"))
9241        } else {
9242            Ok(())
9243        }
9244    }
9245
9246    fn spec() -> ModuleSpec {
9247        ModuleSpec {
9248            module_id: "spawn-record-worker".into(),
9249            #[cfg(unix)]
9250            program: "/bin/sleep".into(),
9251            #[cfg(windows)]
9252            program: std::env::current_exe().unwrap(),
9253            #[cfg(unix)]
9254            args: vec!["60".into()],
9255            #[cfg(windows)]
9256            args: vec![
9257                "--exact".into(),
9258                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9259                "--nocapture".into(),
9260            ],
9261            env: vec![],
9262            reserved: false,
9263            reserved_prefixes: vec![],
9264            protocol: ModuleProtocol::None,
9265            overlap: Default::default(),
9266        }
9267    }
9268
9269    #[tokio::test]
9270    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9271        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9272        let supervisor =
9273            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9274                .with_live_children_record(home.join("live-children.json"));
9275        crate::live_children::fail_next_file_sync();
9276        let result = supervisor.spawn(spec());
9277        let failed = result.is_err();
9278        if let Ok(module) = result {
9279            module.stop().await.unwrap();
9280        }
9281        assert!(failed, "record failure must fail spawn");
9282        let pid = LAST_PID.get().expect("the real OS child was created");
9283        assert!(
9284            subc_os::Process::open(pid).unwrap().is_none(),
9285            "child {pid} survived rejected admission or was not reaped"
9286        );
9287    }
9288
9289    #[cfg(target_os = "macos")]
9290    #[test]
9291    #[ignore = "manual on-device latency measurement"]
9292    fn durable_record_write_latency_50() {
9293        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9294        let path = home.join("live-children.json");
9295        let children = vec![crate::live_children::LiveChild {
9296            module_id: "latency-worker".into(),
9297            pid: std::process::id(),
9298            protocol: ModuleProtocol::None,
9299            start_time: Some(1),
9300            executable: None,
9301            cgroup_name: None,
9302        }];
9303        let mut samples = Vec::new();
9304        for _ in 0..50 {
9305            let start = std::time::Instant::now();
9306            crate::live_children::write_record(&path, &children).unwrap();
9307            samples.push(start.elapsed());
9308        }
9309        samples.sort();
9310        eprintln!(
9311            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9312            (samples[24] + samples[25]) / 2,
9313            samples[47]
9314        );
9315    }
9316
9317    #[cfg(windows)]
9318    #[test]
9319    fn windows_store_touch_fixture() {
9320        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9321            std::fs::write(path, b"module ran").unwrap();
9322        }
9323    }
9324
9325    #[cfg(windows)]
9326    #[tokio::test]
9327    async fn windows_daemon_fixture() {
9328        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9329            return;
9330        };
9331        let mut spec = spec();
9332        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9333        let supervisor =
9334            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9335        let _module = supervisor.spawn(spec).unwrap();
9336    }
9337
9338    #[cfg(windows)]
9339    #[test]
9340    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9341        use std::io::Read;
9342        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9343        // An outer job models a scheduler/service host. The daemon's root and
9344        // root job must nest without requiring breakaway from this job.
9345        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9346        command
9347            .args([
9348                "--exact",
9349                "supervise::spawn_record_tests::windows_daemon_fixture",
9350                "--nocapture",
9351            ])
9352            .env(
9353                "SUBC_TEST_DAEMON_BARRIER",
9354                listener.local_addr().unwrap().to_string(),
9355            );
9356        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9357        listener.set_nonblocking(true).unwrap();
9358        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9359        let mut stream = loop {
9360            if let Ok((stream, _)) = listener.accept() {
9361                break stream;
9362            }
9363            assert!(
9364                daemon.child.try_wait().unwrap().is_none(),
9365                "daemon failed before rendezvous"
9366            );
9367            assert!(
9368                std::time::Instant::now() < deadline,
9369                "daemon did not reach CreateProcess failpoint"
9370            );
9371            std::thread::yield_now();
9372        };
9373        stream
9374            .set_read_timeout(Some(Duration::from_secs(30)))
9375            .unwrap();
9376        let mut pid = [0; 4];
9377        stream.read_exact(&mut pid).unwrap();
9378        let pid = u32::from_ne_bytes(pid);
9379        assert!(subc_jobobject::process_exists(pid));
9380        // Terminate only the daemon, not its outer job: otherwise the host job
9381        // would also kill a child whose daemon never joined its root job.
9382        daemon.child.kill().unwrap();
9383        daemon.child.wait().unwrap();
9384        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9385        if !exited {
9386            daemon.job.terminate().unwrap();
9387        }
9388        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9389    }
9390
9391    #[cfg(windows)]
9392    #[tokio::test]
9393    async fn windows_containment_failure_never_resumes() {
9394        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9395        let sentinel = home.join("store-touched");
9396        let mut spec = spec();
9397        spec.env
9398            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9399        spec.env.push((
9400            "SUBC_TEST_STORE_TOUCH".into(),
9401            sentinel.display().to_string(),
9402        ));
9403        let supervisor =
9404            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9405        let result = supervisor.spawn(spec);
9406        // A version that resumes despite assignment failure must have time to
9407        // execute its sentinel before dropping the returned module kills it.
9408        if let Ok(module) = result {
9409            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9410                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9411                    tokio::task::yield_now().await;
9412                }
9413            })
9414            .await;
9415            let _ = module.stop().await;
9416            panic!(
9417                "assignment failure resumed a module; store touched={}",
9418                sentinel.exists()
9419            );
9420        }
9421        assert!(
9422            !sentinel.exists(),
9423            "refused module executed store-touch sentinel"
9424        );
9425        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9426    }
9427
9428    #[cfg(windows)]
9429    #[tokio::test]
9430    async fn windows_nested_daemon_fixture() {
9431        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9432            return;
9433        };
9434        let mut spec = spec();
9435        spec.env.push((
9436            "SUBC_TEST_STORE_TOUCH".into(),
9437            sentinel.to_str().unwrap().into(),
9438        ));
9439        let supervisor =
9440            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9441        let module = supervisor
9442            .spawn(spec)
9443            .expect("per-module job must nest below daemon and host jobs");
9444        tokio::time::timeout(Duration::from_secs(10), async {
9445            while !PathBuf::from(&sentinel).exists() {
9446                tokio::task::yield_now().await;
9447            }
9448        })
9449        .await
9450        .unwrap();
9451        module.stop().await.unwrap();
9452    }
9453
9454    #[cfg(windows)]
9455    #[test]
9456    fn windows_module_job_nests_under_host_job() {
9457        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9458        let sentinel = home.join("nested-store-touched");
9459        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9460        command
9461            .args([
9462                "--exact",
9463                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9464                "--nocapture",
9465            ])
9466            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9467        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9468        assert!(daemon.child.wait().unwrap().success());
9469        assert!(sentinel.exists(), "nested protocol:none module never ran");
9470    }
9471}
9472
9473#[cfg(target_os = "linux")]
9474pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9475    use subc_cgroup::KillOutcome;
9476    match subc_cgroup::kill_module(placement, module_id) {
9477        KillOutcome::Killed => {}
9478        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9479            debug!(
9480                module_id,
9481                "cgroup tree kill unavailable; using direct-child kill"
9482            );
9483        }
9484        KillOutcome::IoError { path, error } => {
9485            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9486        }
9487    }
9488}
9489
9490/// Assign a freshly spawned Windows child without ever resuming on failure.
9491///
9492/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9493/// child assigned **while it is still suspended** (step 1 is
9494/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9495/// the child's roster admission is durable. Failed containment kills and reaps
9496/// the suspended child instead of running an unprotected module.
9497#[cfg(windows)]
9498fn contain_spawned_child(
9499    child: &mut Child,
9500    spec: &ModuleSpec,
9501) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9502    let assignment = || {
9503        let job = subc_jobobject::JobObject::new()?;
9504        #[cfg(test)]
9505        spawn_record_tests::assignment_failpoint(spec)?;
9506        job.assign(child)?;
9507        Ok::<_, io::Error>(job)
9508    };
9509    match assignment() {
9510        Ok(job) => Ok(Some(job)),
9511        Err(source) => {
9512            kill_and_reap_rejected_child(child);
9513            Err(SuperviseError::Spawn {
9514                program: spec.program.clone(),
9515                source: io::Error::other(format!(
9516                    "Windows module job containment failed: {source}"
9517                )),
9518                cgroup_path: None,
9519            })
9520        }
9521    }
9522}
9523
9524/// Rejection is synchronous: do not return a spawn error while the child can
9525/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9526fn kill_and_reap_rejected_child(child: &mut Child) {
9527    #[cfg(unix)]
9528    if let Some(pid) = child
9529        .id()
9530        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9531    {
9532        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9533    }
9534    let _ = child.start_kill();
9535    while matches!(child.try_wait(), Ok(None)) {
9536        std::thread::yield_now();
9537    }
9538}
9539
9540/// Resume a suspended child, killing it if it cannot be started.
9541///
9542/// A suspended process holds a pid and does nothing, so there is no useful
9543/// state to return: the caller gets an error and the spawn fails.
9544#[cfg(windows)]
9545fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9546    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9547        // Kill it here rather than leaving a suspended process for the caller
9548        // to notice; `kill_on_drop` would eventually do this, but the module
9549        // would have been reported as running in between.
9550        let _ = std::process::Command::new("taskkill.exe")
9551            .args(["/PID", &pid.to_string(), "/T", "/F"])
9552            .stdin(Stdio::null())
9553            .stdout(Stdio::null())
9554            .stderr(Stdio::null())
9555            .status();
9556        return Err(SuperviseError::Spawn {
9557            program: spec.program.clone(),
9558            source,
9559            cgroup_path: None,
9560        });
9561    }
9562    Ok(())
9563}
9564
9565#[cfg(target_os = "linux")]
9566fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9567    match placement.remove_module(module_id) {
9568        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9569        Err(error) => warn!(
9570            module_id,
9571            error = %error,
9572            "could not remove module cgroup after process exit; continuing teardown"
9573        ),
9574    }
9575}
9576
9577#[cfg(target_os = "linux")]
9578async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9579    // Reaping the direct child is not proof its descendants exited. End the
9580    // residual tree and wait for the kernel's population fact before rmdir;
9581    // otherwise a successful parent wait leaks a directory on each restart.
9582    if matches!(
9583        subc_cgroup::kill_module(Some(placement), module_id),
9584        subc_cgroup::KillOutcome::Killed
9585    ) {
9586        if let Ok(path) = placement.module_path(module_id) {
9587            while std::fs::read_to_string(path.join("cgroup.events"))
9588                .ok()
9589                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9590            {
9591                sleep(Duration::from_millis(1)).await;
9592            }
9593        }
9594    }
9595    remove_module_cgroup(placement, module_id);
9596}
9597
9598#[cfg(target_os = "linux")]
9599fn apply_cgroup_placement(
9600    command: &mut Command,
9601    spec: &ModuleSpec,
9602    path: &std::path::Path,
9603) -> Result<(), SuperviseError> {
9604    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9605        module_id: spec.module_id.clone(),
9606        source,
9607    })
9608}
9609
9610fn capture_retention(spec: &ModuleSpec) -> Retention {
9611    let defaults = Retention::default();
9612    let value = |name: &str| {
9613        spec.env
9614            .iter()
9615            .rev()
9616            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9617    };
9618    Retention {
9619        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9620            .and_then(|value| value.parse().ok())
9621            .unwrap_or(defaults.max_file_mb),
9622        keep: value(CAPTURE_KEEP_ENV)
9623            .and_then(|value| value.parse().ok())
9624            .unwrap_or(defaults.keep),
9625        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9626            .and_then(|value| value.parse().ok())
9627            .unwrap_or(defaults.max_age_days),
9628    }
9629}
9630
9631/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9632/// module's registration to the exact process the supervisor spawned.
9633fn generate_launch_nonce() -> Result<String, SuperviseError> {
9634    let mut bytes = [0u8; 32];
9635    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9636        reason: source.to_string(),
9637    })?;
9638    let mut hex = String::with_capacity(64);
9639    for b in bytes {
9640        use std::fmt::Write;
9641        let _ = write!(hex, "{b:02x}");
9642    }
9643    Ok(hex)
9644}
9645
9646/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9647/// signal about how many leading bytes matched.
9648fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9649    if a.len() != b.len() {
9650        return false;
9651    }
9652    let mut diff = 0u8;
9653    for (x, y) in a.iter().zip(b.iter()) {
9654        diff |= x ^ y;
9655    }
9656    diff == 0
9657}
9658
9659/// The kernel's image after an acknowledged exec, shared by ordinary launches
9660/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9661/// not identities inferred from a configured pathname.
9662#[cfg(not(windows))]
9663fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9664    let process = subc_os::Process::open(pid).ok().flatten()?;
9665    #[cfg(target_os = "linux")]
9666    return sample_after_linux_exec(
9667        || {
9668            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9669            let committed = linux_exec_committed(&stat);
9670            #[cfg(test)]
9671            spawn_image_tests::report_exec_state(committed);
9672            committed
9673        },
9674        || process.observe(),
9675    );
9676    #[cfg(not(target_os = "linux"))]
9677    process.observe()
9678}
9679
9680/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9681/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9682/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9683#[cfg(target_os = "linux")]
9684fn linux_exec_committed(stat: &str) -> Option<bool> {
9685    const PF_FORKNOEXEC: u64 = 0x40;
9686    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9687    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9688}
9689
9690#[cfg(target_os = "linux")]
9691fn sample_after_linux_exec(
9692    mut exec_committed: impl FnMut() -> Option<bool>,
9693    observe: impl FnOnce() -> Option<subc_os::Observation>,
9694) -> Option<subc_os::Observation> {
9695    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9696    while !exec_committed()? {
9697        // A failed or indefinitely stalled exec must not publish the inherited
9698        // image as authority to signal a future orphan. Missing identity keeps
9699        // the existing conservative orphan-sweep behavior.
9700        if std::time::Instant::now() >= deadline {
9701            return None;
9702        }
9703        std::thread::sleep(Duration::from_micros(50));
9704    }
9705    observe()
9706}
9707
9708#[cfg(all(test, target_os = "linux"))]
9709mod spawn_image_tests {
9710    use super::*;
9711    use std::cell::{Cell, RefCell};
9712
9713    enum ExecEvent {
9714        Uncommitted,
9715        Image(Option<subc_os::Observation>),
9716    }
9717
9718    thread_local! {
9719        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9720    }
9721
9722    pub(super) fn report_exec_state(committed: Option<bool>) {
9723        if committed == Some(false) {
9724            EXEC_EVENTS.with(|events| {
9725                if let Some(events) = events.borrow_mut().take() {
9726                    let _ = events.send(ExecEvent::Uncommitted);
9727                }
9728            });
9729        }
9730    }
9731
9732    #[test]
9733    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9734        use std::io::{Read, Write};
9735        use std::os::fd::AsRawFd;
9736
9737        let descriptor = std::fs::File::open("/dev/null").unwrap();
9738        let mut command = std::process::Command::new("/bin/sleep");
9739        command.arg("30");
9740        let (mut ready, mut release) =
9741            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9742                .unwrap();
9743        let spawn = std::thread::spawn(move || command.spawn());
9744        let mut message = [0u8; 8];
9745        ready.read_exact(&mut message).unwrap();
9746        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9747        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9748        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9749        let (events, received) = std::sync::mpsc::channel();
9750        let observer = std::thread::spawn(move || {
9751            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9752            let image = observe_spawned_image(pid);
9753            let _ = events.send(ExecEvent::Image(image));
9754        });
9755        // Release only once the production observer has seen the uncommitted
9756        // exec, or has incorrectly returned the inherited image. Either way,
9757        // release and reap before asserting so a regression cannot leak a child.
9758        let first = received.recv_timeout(Duration::from_secs(5));
9759        release.write_all(&[1]).unwrap();
9760        let mut child = spawn.join().unwrap().unwrap();
9761        let after = match first {
9762            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9763                Ok(ExecEvent::Image(image)) => image,
9764                _ => None,
9765            },
9766            Ok(ExecEvent::Image(image)) => image,
9767            Err(_) => None,
9768        };
9769        observer.join().unwrap();
9770        child.kill().unwrap();
9771        child.wait().unwrap();
9772
9773        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9774        assert_eq!(
9775            inherited.unwrap().executable,
9776            subc_os::file_identity(&std::env::current_exe().unwrap())
9777        );
9778        assert_eq!(
9779            after.unwrap().executable,
9780            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9781        );
9782    }
9783
9784    #[test]
9785    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9786        let committed = Cell::new(false);
9787        let inherited = subc_os::Observation {
9788            start_time: 23,
9789            executable: Some(subc_os::FileIdentity {
9790                device: 29,
9791                inode: 31,
9792            }),
9793        };
9794        let module = subc_os::Observation {
9795            start_time: 23,
9796            executable: Some(subc_os::FileIdentity {
9797                device: 29,
9798                inode: 37,
9799            }),
9800        };
9801        let polls = Cell::new(0);
9802        let image = sample_after_linux_exec(
9803            || {
9804                polls.set(polls.get() + 1);
9805                // Hold the inherited image through the first poll, as in the
9806                // posix_spawn parent-wake window, then commit the module image.
9807                committed.set(polls.get() == 2);
9808                Some(committed.get())
9809            },
9810            || Some(if committed.get() { module } else { inherited }),
9811        );
9812        assert_eq!(image, Some(module));
9813        assert_eq!(polls.get(), 2);
9814    }
9815
9816    #[test]
9817    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9818        assert_eq!(
9819            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9820            Some(false)
9821        );
9822        assert_eq!(
9823            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9824            Some(true)
9825        );
9826        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9827        assert_eq!(linux_exec_committed("17 (a) R"), None);
9828    }
9829
9830    #[test]
9831    fn unreadable_exec_state_never_samples_an_inherited_image() {
9832        assert_eq!(
9833            sample_after_linux_exec(
9834                || None,
9835                || panic!("must not sample before exec confirmation")
9836            ),
9837            None
9838        );
9839    }
9840
9841    #[test]
9842    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9843        assert_eq!(
9844            sample_after_linux_exec(
9845                || Some(false),
9846                || panic!("must not sample after an exec timeout")
9847            ),
9848            None
9849        );
9850    }
9851}
9852
9853/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9854/// different executable after the no-delete capture was taken.
9855#[cfg(windows)]
9856fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9857    let mut program = spec.program.clone();
9858    if program.is_absolute() || program.components().count() > 1 {
9859        // An explicit path can name a PE image with no extension. Try that
9860        // exact file first, as Command does, before falling back to .exe.
9861        match subc_os::ExecutableCapture::open(&program) {
9862            Ok(capture) => return Ok(capture),
9863            Err(error)
9864                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9865            Err(error) => return Err(error),
9866        }
9867        program.set_extension("exe");
9868        return subc_os::ExecutableCapture::open(&program);
9869    }
9870    if program.extension().is_none() {
9871        program.set_extension("exe");
9872    }
9873    let custom_path = spec
9874        .env
9875        .iter()
9876        .rev()
9877        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9878    let mut directories = Vec::new();
9879    // These are Windows' default executable search locations. A configured
9880    // PATH still overrides the inherited PATH, not the system directories.
9881    if let Some(parent) = std::env::current_exe()?.parent() {
9882        directories.push(parent.to_path_buf());
9883    }
9884    directories.push(std::env::current_dir()?);
9885    if let Some(windows) = std::env::var_os("SystemRoot") {
9886        let windows = PathBuf::from(windows);
9887        directories.push(windows.join("System32"));
9888        directories.push(windows.clone());
9889    }
9890    let path = custom_path
9891        .map(|(_, value)| std::ffi::OsString::from(value))
9892        .or_else(|| std::env::var_os("PATH"));
9893    if let Some(path) = path {
9894        directories.extend(std::env::split_paths(&path));
9895    }
9896    for directory in directories {
9897        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9898            Ok(capture) => return Ok(capture),
9899            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9900            Err(error) => return Err(error),
9901        }
9902    }
9903    Err(io::Error::new(
9904        io::ErrorKind::NotFound,
9905        "resolved executable not found",
9906    ))
9907}
9908
9909fn spawn_and_mark_running(
9910    spec: &ModuleSpec,
9911    runtime: &SupervisorRuntimeConfig,
9912    snapshot: &SharedSnapshot,
9913) -> Result<SupervisedChild, SuperviseError> {
9914    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9915    let registration_watch = watch_registration(&runtime.registry, spec);
9916    let child = spawn_child(
9917        spec,
9918        runtime.connection_file_path.as_deref(),
9919        runtime.supervisor_handle.as_ref(),
9920        &runtime.stderr_ring,
9921        runtime.capture_logs_dir.as_deref(),
9922        &runtime.child_roster,
9923        #[cfg(target_os = "linux")]
9924        runtime.cgroup_placement.as_ref(),
9925    )?;
9926    let mut child = child;
9927    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
9928    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9929    if let Some(spawn) = activity_spawn {
9930        spawn.complete(snapshot);
9931    }
9932    Ok(child)
9933}
9934
9935struct RegistrationWatch {
9936    changes: watch::Receiver<u64>,
9937    baseline: u64,
9938}
9939
9940// Capture before spawning: a fast child's accepted HELLO can disappear from the
9941// active registry before the monitor first runs. A pre-existing registration
9942// must not satisfy a new process's deadline either.
9943fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
9944    if spec.protocol != ModuleProtocol::Subc {
9945        return None;
9946    }
9947    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
9948    let baseline = *changes.borrow();
9949    Some(RegistrationWatch { changes, baseline })
9950}
9951
9952fn arm_registration_deadline(
9953    child: &mut SupervisedChild,
9954    budget: Duration,
9955    registration_watch: Option<RegistrationWatch>,
9956) {
9957    child.registration_deadline =
9958        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
9959    child.registration_watch = registration_watch;
9960}
9961
9962async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
9963    #[cfg(unix)]
9964    {
9965        // Children lead their own process groups. Signal the whole group so a
9966        // helper stuck before HELLO cannot survive its supervised parent.
9967        let group = unreaped_registration_group(child)?;
9968        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
9969        // Process-group safety rule: only signal a group while its leader is
9970        // our unreaped child. Even a zombie leader reserves its pid/group id;
9971        // reaping it before KILL would let that id name an unrelated group.
9972        // Sleep without waiting so TERM-ignoring helpers get the full grace.
9973        sleep(crate::child_roster::TERM_TO_KILL).await;
9974        let group = unreaped_registration_group(child)?;
9975        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
9976    }
9977    // Windows terminates the containing Job Object as well as the direct
9978    // child. Linux also kills the child's cgroup when placement is enabled.
9979    child.start_kill()?;
9980    child.wait().await?;
9981    Ok(())
9982}
9983
9984#[cfg(unix)]
9985fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
9986    // The group id comes only from the pid recorded at spawn, when the child
9987    // created its own group. Never admit init's group or a negative group id.
9988    let group = i32::try_from(child.pid)
9989        .ok()
9990        .filter(|pid| *pid > 1)
9991        .and_then(rustix::process::Pid::from_raw)
9992        .ok_or_else(|| {
9993            io::Error::new(
9994                io::ErrorKind::InvalidInput,
9995                "invalid supervised process group",
9996            )
9997        })?;
9998    if child.child.id() != Some(child.pid) {
9999        return Err(io::Error::new(
10000            io::ErrorKind::InvalidInput,
10001            "refusing process-group signal after supervised child was reaped",
10002        ));
10003    }
10004    Ok(group)
10005}
10006
10007enum RegistrationWaitOutcome {
10008    Registered,
10009    Exited(ExitReport),
10010    TimedOut,
10011}
10012
10013struct ReloadRegistrationFailure {
10014    exit_report: ExitReport,
10015    reason: String,
10016}
10017
10018#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10019enum BusyGaugeObservation {
10020    Quiescent,
10021    Busy,
10022    Omitted,
10023}
10024
10025/// What one health answer said about one declared busy gauge.
10026#[derive(Debug, Clone, PartialEq, Eq)]
10027enum GaugeVerdict {
10028    /// The gauge read 0: this gauge is not what holds the drain.
10029    Zero,
10030    /// The gauge read a positive count.
10031    NonZero(u64),
10032    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10033    /// a negative number). It counts as busy, because the daemon cannot tell
10034    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10035    NotACount(String),
10036    /// The health answer had no metrics object, or one without this gauge.
10037    Missing,
10038    /// The probe produced no health answer to read; carries the probe error's
10039    /// evidence label (`no-answer` for a probe that timed out).
10040    ProbeFailed(&'static str),
10041}
10042
10043impl GaugeVerdict {
10044    fn reads_quiet(&self) -> bool {
10045        matches!(self, Self::Zero)
10046    }
10047}
10048
10049impl fmt::Display for GaugeVerdict {
10050    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10051        match self {
10052            Self::Zero => f.write_str("zero"),
10053            Self::NonZero(value) => write!(f, "non_zero({value})"),
10054            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10055            Self::Missing => f.write_str("missing_from_health_body"),
10056            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10057        }
10058    }
10059}
10060
10061/// The longest JSON text kept for a gauge value that is not a count, so a
10062/// module reporting a large object as a gauge cannot flood the drain log.
10063const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10064
10065/// Read each declared gauge out of one health answer's metrics.
10066fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10067    let metrics = metrics.and_then(Value::as_object);
10068    gauges
10069        .iter()
10070        .map(
10071            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10072                None => GaugeVerdict::Missing,
10073                Some(value) => match value.as_u64() {
10074                    Some(0) => GaugeVerdict::Zero,
10075                    Some(count) => GaugeVerdict::NonZero(count),
10076                    None => {
10077                        let mut text = value.to_string();
10078                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10079                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10080                            while !text.is_char_boundary(end) {
10081                                end -= 1;
10082                            }
10083                            text.truncate(end);
10084                            text.push_str("...");
10085                        }
10086                        GaugeVerdict::NotACount(text)
10087                    }
10088                },
10089            },
10090        )
10091        .collect()
10092}
10093
10094/// The drain's overall reading of one health answer: omitted when any
10095/// declared gauge is missing, quiescent only when every gauge reads 0.
10096fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10097    if verdicts
10098        .iter()
10099        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10100    {
10101        BusyGaugeObservation::Omitted
10102    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10103        BusyGaugeObservation::Quiescent
10104    } else {
10105        BusyGaugeObservation::Busy
10106    }
10107}
10108
10109/// One declared gauge's history over a drain: the latest verdict on it, and
10110/// the latest value actually read for it (which a later failed probe or
10111/// missing gauge does not erase). Times are offsets from the start of the
10112/// drain's wait.
10113#[derive(Debug, Clone, Default, PartialEq, Eq)]
10114struct GaugeTrace {
10115    latest: Option<(GaugeVerdict, Duration)>,
10116    last_value: Option<(GaugeVerdict, Duration)>,
10117}
10118
10119/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10120/// its ceiling can say in its timeout line which declared gauge held it and
10121/// why. Without it, a gauge that read busy, a gauge the module never
10122/// reported, and a probe that never got an answer all looked the same: a
10123/// drain that ended at its ceiling with nothing in flight.
10124#[derive(Debug, Clone, Default, PartialEq, Eq)]
10125struct DrainGaugeTrace {
10126    /// Health probes the drain sent.
10127    probes: u32,
10128    /// Each declared gauge, in declaration order.
10129    gauges: Vec<(String, GaugeTrace)>,
10130}
10131
10132impl DrainGaugeTrace {
10133    fn new(gauges: &[String]) -> Self {
10134        Self {
10135            probes: 0,
10136            gauges: gauges
10137                .iter()
10138                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10139                .collect(),
10140        }
10141    }
10142
10143    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10144    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10145        self.probes = self.probes.saturating_add(1);
10146        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10147            trace.latest = Some((verdict.clone(), at));
10148            if matches!(
10149                verdict,
10150                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10151            ) {
10152                trace.last_value = Some((verdict.clone(), at));
10153            }
10154        }
10155    }
10156}
10157
10158/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10159/// declared gauge, separated by ` | `. A value is shown as its verdict
10160/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10161/// and whether that counted as busy.
10162impl fmt::Display for DrainGaugeTrace {
10163    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10164        if self.gauges.is_empty() {
10165            return f.write_str("none declared");
10166        }
10167        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10168            if index > 0 {
10169                f.write_str(" | ")?;
10170            }
10171            write!(f, "{name}: ")?;
10172            match &trace.latest {
10173                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10174                None => f.write_str("never probed")?,
10175            }
10176            match &trace.last_value {
10177                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10178                None => f.write_str("; no value ever read")?,
10179            }
10180        }
10181        Ok(())
10182    }
10183}
10184
10185pub(crate) fn declared_busy_gauges(
10186    registry: &Registry,
10187    module_id: &str,
10188) -> Result<Vec<String>, SuperviseError> {
10189    busy_gauges_of(
10190        registry
10191            .get_module(module_id)
10192            .map_err(SuperviseError::Registry)?,
10193    )
10194}
10195
10196/// [`declared_busy_gauges`] for the registration a connection holds, in any
10197/// slot: after cutover the incumbent is no longer the id's active
10198/// registration, and its own manifest is the one that names its gauges.
10199fn declared_busy_gauges_for_connection(
10200    registry: &Registry,
10201    connection_id: ConnectionId,
10202) -> Result<Vec<String>, SuperviseError> {
10203    busy_gauges_of(
10204        registry
10205            .get_module_by_connection(connection_id)
10206            .map_err(SuperviseError::Registry)?,
10207    )
10208}
10209
10210fn busy_gauges_of(
10211    registration: Option<crate::registry::ModuleRegistration>,
10212) -> Result<Vec<String>, SuperviseError> {
10213    let Some(registration) = registration else {
10214        return Ok(Vec::new());
10215    };
10216    let Some(self_signals) = registration.manifest.self_signals else {
10217        return Ok(Vec::new());
10218    };
10219
10220    let mut gauges = Vec::new();
10221    for declaration in self_signals {
10222        if declaration.kind != SelfSignalKind::Busy {
10223            continue;
10224        }
10225        match declaration.anchored_to {
10226            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10227                gauges.extend(declared)
10228            }
10229            _ => {
10230                // An invalid Busy anchor is fail-safe: the empty name cannot be
10231                // present in a conforming health report, so this drain stays busy.
10232                gauges.push(String::new());
10233            }
10234        }
10235    }
10236    Ok(gauges)
10237}
10238
10239/// Wait for `endpoint` to have nothing in flight and, when the module declares
10240/// busy gauges, for a health probe to report them quiet. The probe is addressed
10241/// by `scope`: a swap's superseded incumbent must be asked about its own
10242/// gauges, and by module id the probe would reach the promoted candidate.
10243/// While the gauges read busy, the probe repeats every
10244/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10245/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10246/// timeout line.
10247#[allow(clippy::too_many_arguments)]
10248async fn wait_for_forwarding_quiescence(
10249    forwarding: &ForwardingTable,
10250    module_id: &str,
10251    runtime: &SupervisorRuntimeConfig,
10252    endpoint: crate::ModuleEndpointId,
10253    deadline: Instant,
10254    busy_gauges: &[String],
10255    scope: DrainScope,
10256    trace: &mut DrainGaugeTrace,
10257) -> Result<bool, SuperviseError> {
10258    let wait_started = Instant::now();
10259    let mut gauges_quiescent = busy_gauges.is_empty();
10260    let mut next_probe_at = wait_started;
10261    let mut omission_reported = false;
10262
10263    loop {
10264        let now = Instant::now();
10265        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10266            let report = match scope {
10267                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10268                DrainScope::Endpoint(endpoint) => {
10269                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10270                }
10271            };
10272            let verdicts = match &report {
10273                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10274                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10275            };
10276            trace.record(&verdicts, wait_started.elapsed());
10277            gauges_quiescent = match report {
10278                Ok(_) => match busy_gauge_observation(&verdicts) {
10279                    BusyGaugeObservation::Quiescent => true,
10280                    BusyGaugeObservation::Busy => false,
10281                    BusyGaugeObservation::Omitted => {
10282                        // Once per drain: a module that never reports a gauge
10283                        // it declared would otherwise say so on every probe.
10284                        if !omission_reported {
10285                            forwarding
10286                                .counters()
10287                                .increment_drains_with_undeclared_gauge();
10288                            let missing = busy_gauges
10289                                .iter()
10290                                .zip(&verdicts)
10291                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10292                                .map(|(gauge, _)| gauge.as_str())
10293                                .collect::<Vec<_>>()
10294                                .join(",");
10295                            warn!(
10296                                module_id,
10297                                missing_gauges = %missing,
10298                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10299                            );
10300                            omission_reported = true;
10301                        }
10302                        false
10303                    }
10304                },
10305                Err(err) => {
10306                    warn!(
10307                        module_id,
10308                        error = %err,
10309                        "drain health.check did not produce declared busy gauges; treating module as busy"
10310                    );
10311                    false
10312                }
10313            };
10314            // Counted from the probe's completion: the probe above was awaited,
10315            // so no second probe can start while one is still outstanding.
10316            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10317        }
10318
10319        let in_flight = forwarding
10320            .endpoint_in_flight_count(endpoint)
10321            .map_err(SuperviseError::Forwarding)?;
10322        if in_flight == 0 && gauges_quiescent {
10323            return Ok(true);
10324        }
10325
10326        let now = Instant::now();
10327        if now >= deadline {
10328            return Ok(false);
10329        }
10330        let mut wait = deadline
10331            .saturating_duration_since(now)
10332            .min(REGISTRY_RELEASE_POLL);
10333        if !busy_gauges.is_empty() {
10334            wait = wait.min(next_probe_at.saturating_duration_since(now));
10335        }
10336        sleep(wait).await;
10337    }
10338}
10339
10340/// The wait between drain busy-gauge probes: the larger of
10341/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10342/// drain's own in-flight poll interval.
10343fn drain_busy_reprobe_interval() -> Duration {
10344    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10345}
10346
10347/// Logs why a drain gave up: the drain waited its full timeout without the
10348/// module going quiet, and teardown is being forced.
10349///
10350/// The line names two kinds of evidence, because a bare "timed out" hides
10351/// causes that need different fixes:
10352/// - `holdouts`: the requests and routes still open when the timeout hit,
10353///   and on which connections. One stuck call is a module bug; a held stream
10354///   on every session is a module that should end its streams when it
10355///   receives `module.draining`.
10356/// - `gauges`: for each busy gauge the module declares, its latest reading
10357///   and how many probes ran. This separates a module that kept reporting
10358///   itself busy, one whose health answer never contained the gauge, and one
10359///   that never answered the probes, which otherwise all look like a timeout
10360///   with nothing in flight.
10361fn warn_drain_timed_out(
10362    module_id: &str,
10363    drain_timeout: Duration,
10364    reason: RouteCloseReason,
10365    holdouts: &crate::forwarding::DrainHoldouts,
10366    gauges: &DrainGaugeTrace,
10367) {
10368    warn!(
10369        module_id = %module_id,
10370        waited = ?drain_timeout,
10371        ?reason,
10372        held_requests = holdouts.requests,
10373        held_routes = holdouts.routes,
10374        total_routes = holdouts.total_routes,
10375        top_connections = ?holdouts.top_connections,
10376        // `module_channel:corr`, so the module can find each held request
10377        // in its own log; capped, so `held_requests` is the full count.
10378        held = %holdouts
10379            .held
10380            .iter()
10381            .map(|(channel, corr)| format!("{channel}:{corr}"))
10382            .collect::<Vec<_>>()
10383            .join(","),
10384        gauge_probes = gauges.probes,
10385        busy_gauges = %gauges,
10386        "route drain timed out before request quiescence; forcing teardown"
10387    );
10388}
10389
10390/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10391///
10392/// `Ok` is always honest and passed straight through -- the wait actually measured
10393/// in-flight state. `Err` means the wait produced no measurement at all (the
10394/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10395/// constant: the drain did not complete. Never recomputed from route state, never a
10396/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10397fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10398    match wait_result {
10399        Ok(drained) => *drained,
10400        Err(_) => false,
10401    }
10402}
10403
10404fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10405    for released in released_routes {
10406        let frame = match Frame::build_with_version(
10407            released.negotiated_ver,
10408            FrameType::Goodbye,
10409            control_flags(),
10410            released.channel,
10411            released.epoch,
10412            0,
10413            Vec::new(),
10414        ) {
10415            Ok(frame) => frame,
10416            Err(err) => {
10417                warn!(
10418                    route_channel = released.channel,
10419                    error = %err,
10420                    "failed to build supervisor drain route GOODBYE frame"
10421                );
10422                continue;
10423            }
10424        };
10425        if !released.close_on_delivery_failure() {
10426            crate::forwarding::send_module_route_goodbye(
10427                &forwarding.counters(),
10428                &released.sink,
10429                frame,
10430                released.module_id.as_deref(),
10431                "supervisor drain",
10432            );
10433            continue;
10434        }
10435        if let Err(err) = released.sink.try_send(frame) {
10436            warn!(
10437                target_connection_id = released.connection_id.get(),
10438                route_channel = released.channel,
10439                error = %err,
10440                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10441            );
10442            let _ = forwarding.escalate_client_delivery_failure(
10443                released.connection_id,
10444                released.channel,
10445                released.epoch,
10446                CloseReason::new(
10447                    "route_goodbye_delivery_failed",
10448                    format!(
10449                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10450                        released.channel
10451                    ),
10452                ),
10453                crate::forwarding::UndeliveredFrame {
10454                    module_id: released.module_id.as_deref(),
10455                    sink: &released.sink,
10456                },
10457            );
10458        }
10459    }
10460}
10461
10462fn send_module_draining(
10463    module_id: &str,
10464    reason: RouteCloseReason,
10465    deadline_ms: u64,
10466    target: &ModuleDrainTarget,
10467) {
10468    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10469        reason,
10470        deadline_ms,
10471    }) {
10472        Ok(body) => body,
10473        Err(err) => {
10474            warn!(
10475                module_id,
10476                error = %err,
10477                "failed to encode module draining command"
10478            );
10479            return;
10480        }
10481    };
10482    let frame = match Frame::build_with_version(
10483        target.negotiated_ver,
10484        FrameType::Push,
10485        control_flags(),
10486        0,
10487        0,
10488        0,
10489        body,
10490    ) {
10491        Ok(frame) => frame,
10492        Err(err) => {
10493            warn!(
10494                module_id,
10495                error = %err,
10496                "failed to build module draining command frame"
10497            );
10498            return;
10499        }
10500    };
10501    if let Err(err) = target.sink.try_send(frame) {
10502        warn!(
10503            module_id,
10504            target_connection_id = target.endpoint.connection_id.get(),
10505            error = %err,
10506            "module draining command was not delivered to peer"
10507        );
10508    }
10509}
10510
10511/// The channel-0 GOODBYE that tells a module its stop is planned.
10512fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10513    match Frame::build_with_version(
10514        negotiated_ver,
10515        FrameType::Goodbye,
10516        control_flags(),
10517        0,
10518        0,
10519        0,
10520        Vec::new(),
10521    ) {
10522        Ok(frame) => Some(frame),
10523        Err(err) => {
10524            warn!(
10525                module_id,
10526                error = %err,
10527                "failed to build module GOODBYE frame"
10528            );
10529            None
10530        }
10531    }
10532}
10533
10534/// Send every registered module connection its module GOODBYE at daemon
10535/// shutdown, then request that connection's close.
10536///
10537/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10538/// before EOF, so the GOODBYE must reach the socket before the close. A close
10539/// request does not wait for the connection's queued frames: its writer gets a
10540/// bounded grace after the close, is aborted if it overruns it, and the daemon
10541/// process may exit before that grace ends. So with `wait_for_flush`, each
10542/// connection is closed only after its writer has acknowledged writing the
10543/// GOODBYE, or once a short shared budget runs out, so one module that is not
10544/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10545/// are only queued, for a shutdown the operator has told to stop waiting.
10546/// A connection that is already gone is skipped.
10547#[cfg(unix)]
10548async fn send_module_goodbyes_for_daemon_shutdown(
10549    forwarding: &Arc<ForwardingTable>,
10550    reason: &CloseReason,
10551    wait_for_flush: bool,
10552) {
10553    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10554    let targets = match forwarding.module_connections() {
10555        Ok(targets) => targets,
10556        Err(err) => {
10557            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10558            return;
10559        }
10560    };
10561    let deadline = Instant::now() + GOODBYE_BUDGET;
10562    let mut sends = tokio::task::JoinSet::new();
10563    for target in targets {
10564        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10565            continue;
10566        };
10567        if !wait_for_flush {
10568            if let Err(err) = target.sink.try_send(frame) {
10569                debug!(
10570                    module_id = %target.module_id,
10571                    error = %err,
10572                    "shutdown module GOODBYE was not queued"
10573                );
10574            }
10575            continue;
10576        }
10577        let forwarding = Arc::clone(forwarding);
10578        let reason = reason.clone();
10579        sends.spawn(async move {
10580            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10581                Ok(Ok(())) => {}
10582                Ok(Err(err)) => debug!(
10583                    module_id = %target.module_id,
10584                    error = %err,
10585                    "module connection closed before its shutdown GOODBYE was written"
10586                ),
10587                Err(_) => warn!(
10588                    module_id = %target.module_id,
10589                    budget = ?GOODBYE_BUDGET,
10590                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10591                ),
10592            }
10593            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10594        });
10595    }
10596    // Every task ends by the shared deadline, so this wait is bounded too.
10597    while sends.join_next().await.is_some() {}
10598}
10599
10600fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10601    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10602        return;
10603    };
10604    if let Err(err) = target.sink.try_send(frame) {
10605        warn!(
10606            module_id,
10607            target_connection_id = target.endpoint.connection_id.get(),
10608            error = %err,
10609            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10610        );
10611        forwarding.request_connection_close(
10612            target.endpoint.connection_id,
10613            CloseReason::new(
10614                "module_goodbye_delivery_failed",
10615                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10616            ),
10617        );
10618    }
10619}
10620
10621#[derive(Clone, Copy)]
10622struct ForwardingDrainContext<'a> {
10623    spec: &'a ModuleSpec,
10624    runtime: &'a SupervisorRuntimeConfig,
10625    registry: &'a Registry,
10626    scope: DrainScope,
10627}
10628
10629/// Which process a forwarding drain addresses.
10630#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10631enum DrainScope {
10632    /// Whatever endpoint is active for the module id: every plain stop,
10633    /// restart and reload. Also moves the module's state to `Draining`.
10634    Active,
10635    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10636    /// module id would resolve to the promoted candidate and leave neither
10637    /// process routable. The module's state is left alone, since the promoted
10638    /// candidate is what it describes and that process is running.
10639    Endpoint(crate::ModuleEndpointId),
10640}
10641
10642/// Whether a child being drained has already been asked to stop by the time
10643/// its drain wait starts.
10644///
10645/// The drain wait is the same budget whatever this says. What it decides is
10646/// whether the supervisor must ask by signal before that wait begins: a child
10647/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10648/// healthy or not.
10649#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10650enum StopNotice {
10651    /// The module was sent `module.draining` and a module GOODBYE over its own
10652    /// registered connection, and stops itself.
10653    SentOverConnection,
10654    /// The forwarding drain found no registered connection for the module: a
10655    /// subc child spawned moments ago that has not sent HELLO yet, or a
10656    /// `protocol: "none"` child, which never registers.
10657    NoConnection,
10658    /// This path sends nothing over the module's connection: the supervisor has
10659    /// no forwarding table, or the caller stops the child without a forwarding
10660    /// drain.
10661    NotSent,
10662}
10663
10664async fn begin_forwarding_drain(
10665    spec: &ModuleSpec,
10666    runtime: &SupervisorRuntimeConfig,
10667    registry: &Registry,
10668    snapshot: &SharedSnapshot,
10669    enabled: Option<bool>,
10670    reason: RouteCloseReason,
10671) -> Result<StopNotice, SuperviseError> {
10672    let Some(forwarding) = runtime.forwarding.as_ref() else {
10673        return Err(SuperviseError::ReloadUnavailable {
10674            module_id: spec.module_id.clone(),
10675            reason: "supervisor was not configured with a forwarding table".to_string(),
10676        });
10677    };
10678
10679    begin_forwarding_drain_with(
10680        forwarding,
10681        ForwardingDrainContext {
10682            spec,
10683            runtime,
10684            registry,
10685            scope: DrainScope::Active,
10686        },
10687        snapshot,
10688        enabled,
10689        reason,
10690        runtime.drain_timeout,
10691    )
10692    .await
10693}
10694
10695async fn begin_forwarding_drain_if_configured(
10696    spec: &ModuleSpec,
10697    runtime: &SupervisorRuntimeConfig,
10698    registry: &Registry,
10699    snapshot: &SharedSnapshot,
10700    enabled: Option<bool>,
10701    reason: RouteCloseReason,
10702) -> Result<StopNotice, SuperviseError> {
10703    begin_forwarding_drain_with_timeout(
10704        spec,
10705        runtime,
10706        registry,
10707        snapshot,
10708        enabled,
10709        reason,
10710        runtime.drain_timeout,
10711    )
10712    .await
10713}
10714
10715/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10716/// budget, for paths where the operator overrides the module's configured one
10717/// (`supervisor.restart{drain_timeout_ms}`).
10718async fn begin_forwarding_drain_with_timeout(
10719    spec: &ModuleSpec,
10720    runtime: &SupervisorRuntimeConfig,
10721    registry: &Registry,
10722    snapshot: &SharedSnapshot,
10723    enabled: Option<bool>,
10724    reason: RouteCloseReason,
10725    drain_timeout: Duration,
10726) -> Result<StopNotice, SuperviseError> {
10727    let Some(forwarding) = runtime.forwarding.as_ref() else {
10728        return Ok(StopNotice::NotSent);
10729    };
10730
10731    begin_forwarding_drain_with(
10732        forwarding,
10733        ForwardingDrainContext {
10734            spec,
10735            runtime,
10736            registry,
10737            scope: DrainScope::Active,
10738        },
10739        snapshot,
10740        enabled,
10741        reason,
10742        drain_timeout,
10743    )
10744    .await
10745}
10746
10747async fn begin_forwarding_drain_with(
10748    forwarding: &ForwardingTable,
10749    context: ForwardingDrainContext<'_>,
10750    snapshot: &SharedSnapshot,
10751    enabled: Option<bool>,
10752    reason: RouteCloseReason,
10753    drain_timeout: Duration,
10754) -> Result<StopNotice, SuperviseError> {
10755    let ForwardingDrainContext {
10756        spec,
10757        runtime,
10758        registry,
10759        scope,
10760    } = context;
10761    debug_assert_ne!(reason, RouteCloseReason::Crash);
10762    let terminal = matches!(reason, RouteCloseReason::Disable);
10763    let drain_started_at = Instant::now();
10764    let drain_deadline = drain_started_at + drain_timeout;
10765    let deadline_ms =
10766        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10767    let busy_gauges = match scope {
10768        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10769        DrainScope::Endpoint(endpoint) => {
10770            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10771        }
10772    };
10773
10774    // Admission gate first: route.open/commit and route REQUEST admission are closed
10775    // before the first quiescence check, so the outstanding count can only fall.
10776    let gate_started = Instant::now();
10777    let drain_target = match scope {
10778        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10779        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10780    }
10781    .map_err(SuperviseError::Forwarding)?;
10782    // The instant admission closed, and how long taking the forwarding write
10783    // lock to close it took. The timeout line reports only the quiescence
10784    // wait, so without this a drain that started late looked like one that
10785    // started on time.
10786    info!(
10787        module_id = %spec.module_id,
10788        ?reason,
10789        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10790        connected = drain_target.is_some(),
10791        "module drain began; route admission closed"
10792    );
10793    if scope == DrainScope::Active {
10794        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10795            state.state = ModuleState::Draining;
10796            state.draining_to_replace =
10797                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10798            if let Some(enabled) = enabled {
10799                state.enabled = enabled;
10800            }
10801        })?;
10802    }
10803
10804    let Some(target) = drain_target.as_ref() else {
10805        // Nothing was sent: the module has no registered connection to carry
10806        // `module.draining` or a GOODBYE. The caller must not assume the child
10807        // was asked to stop.
10808        return Ok(StopNotice::NoConnection);
10809    };
10810    {
10811        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10812        #[cfg(feature = "test-support")]
10813        forwarding.wait_before_route_closing_for_test().await;
10814        let routes = forwarding
10815            .endpoint_routes(target.endpoint)
10816            .map_err(SuperviseError::Forwarding)?;
10817        let routes_notified = routes.len();
10818        crate::control::send_route_control_pushes(
10819            forwarding,
10820            routes.clone(),
10821            ClientControlPush::RouteClosing {
10822                module_id: spec.module_id.clone(),
10823                channels: Vec::new(),
10824                reason,
10825            },
10826        );
10827        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10828
10829        // `route.closing` was just sent above: from here on every return path,
10830        // including an early one, MUST send `route.closed` before propagating
10831        // anything else. A client holds `closing` as a promise that a verdict is
10832        // coming; leaving early without `closed` strands it waiting forever, since
10833        // `closing` carries no timeout of its own.
10834        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10835        let wait_result = wait_for_forwarding_quiescence(
10836            forwarding,
10837            &spec.module_id,
10838            runtime,
10839            target.endpoint,
10840            drain_deadline,
10841            &busy_gauges,
10842            scope,
10843            &mut gauge_trace,
10844        )
10845        .await;
10846        let drained = drained_after_quiescence_wait(&wait_result);
10847        if let Err(err) = &wait_result {
10848            error!(
10849                module_id = %spec.module_id,
10850                ?reason,
10851                error = %err,
10852                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10853            );
10854        } else if !drained {
10855            // Read before teardown releases the routes.
10856            let holdouts = forwarding
10857                .endpoint_drain_holdouts(target.endpoint)
10858                .unwrap_or_default();
10859            warn_drain_timed_out(
10860                &spec.module_id,
10861                drain_timeout,
10862                reason,
10863                &holdouts,
10864                &gauge_trace,
10865            );
10866        }
10867        crate::control::send_route_control_pushes(
10868            forwarding,
10869            routes,
10870            ClientControlPush::RouteClosed {
10871                module_id: spec.module_id.clone(),
10872                channels: Vec::new(),
10873                reason,
10874                drained,
10875                abandoned: target.abandoned_bindings.len() as u32,
10876                excluded_subscriptions: target.excluded_subscriptions,
10877                terminal: Some(terminal),
10878            },
10879        );
10880        wait_result?;
10881
10882        // `route.closed` has now been sent unconditionally above. From here the
10883        // remaining steps are cleanup (route + module GOODBYE) rather than a
10884        // promise the client is waiting on, but a lock-poisoned
10885        // `release_module_endpoint_routes` would otherwise skip the module
10886        // GOODBYE silently too -- send it before propagating the error.
10887        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10888            Ok(routes) => routes,
10889            Err(err) => {
10890                warn!(
10891                    module_id = %spec.module_id,
10892                    ?reason,
10893                    error = %err,
10894                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10895                );
10896                send_module_goodbye(&spec.module_id, forwarding, target);
10897                return Err(SuperviseError::Forwarding(err));
10898            }
10899        };
10900        let route_goodbye_count = released_routes.len();
10901        send_route_goodbyes(forwarding, released_routes);
10902        send_module_goodbye(&spec.module_id, forwarding, target);
10903
10904        // The drain's happy path was previously silent: every emission above is
10905        // best-effort with only its failure arm logged, so "were consumers told"
10906        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10907        // hang where the open question was exactly whether teardown notice went
10908        // out). One summary line makes that class decidable in one grep.
10909        info!(
10910            module_id = %spec.module_id,
10911            ?reason,
10912            routes_notified,
10913            route_goodbyes = route_goodbye_count,
10914            abandoned_reservations = target.abandoned_bindings.len(),
10915            excluded_subscriptions = target.excluded_subscriptions,
10916            drained,
10917            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10918        );
10919    }
10920
10921    Ok(StopNotice::SentOverConnection)
10922}
10923
10924/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10925/// the only slot a plain (non-swap) spawn can register into.
10926async fn wait_for_registration_after_reload(
10927    registry: &Registry,
10928    module_id: &str,
10929    snapshot: &SharedSnapshot,
10930    child: &mut SupervisedChild,
10931    wait: Duration,
10932) -> Result<RegistrationWaitOutcome, SuperviseError> {
10933    wait_for_slot_registration(
10934        registry,
10935        crate::registry::RegistrationSlot::Active(module_id),
10936        module_id,
10937        snapshot,
10938        child,
10939        wait,
10940    )
10941    .await
10942}
10943
10944/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10945///
10946/// Keyed on the slot rather than the bare module id because during a swap the
10947/// id's active slot is already held by the incumbent: an id-keyed wait would
10948/// report the incumbent's registration as the candidate's and a candidate that
10949/// never registers would look registered. A swap candidate waits on
10950/// `crate::registry::RegistrationSlot::Candidate`.
10951async fn wait_for_slot_registration(
10952    registry: &Registry,
10953    slot: crate::registry::RegistrationSlot<'_>,
10954    module_id: &str,
10955    snapshot: &SharedSnapshot,
10956    child: &mut SupervisedChild,
10957    wait: Duration,
10958) -> Result<RegistrationWaitOutcome, SuperviseError> {
10959    let deadline = Instant::now() + wait;
10960    loop {
10961        if registry
10962            .registration(slot)
10963            .map_err(SuperviseError::Registry)?
10964            .is_some()
10965        {
10966            return Ok(RegistrationWaitOutcome::Registered);
10967        }
10968
10969        let now = Instant::now();
10970        if now >= deadline {
10971            return Ok(RegistrationWaitOutcome::TimedOut);
10972        }
10973        let remaining = deadline.saturating_duration_since(now);
10974        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10975
10976        tokio::select! {
10977            wait_result = child.wait() => {
10978                let status = wait_result.map_err(|source| SuperviseError::Wait {
10979                    module_id: module_id.to_string(),
10980                    source,
10981                })?;
10982                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10983                    snapshot,
10984                    child,
10985                    &status,
10986                )));
10987            }
10988            _ = sleep(poll) => {}
10989        }
10990    }
10991}
10992
10993fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10994    // A process stopped for missing its first HELLO did not provide service,
10995    // even if its TERM handler exits zero. Reload also uses this classification
10996    // when a replacement exits before registering.
10997    if exit_report.kind != ExitKind::DeliberateSeverance {
10998        exit_report.kind = ExitKind::Crash;
10999    }
11000    exit_report
11001}
11002
11003async fn handle_reload_child_registration_failure(
11004    spec: &ModuleSpec,
11005    runtime: &SupervisorRuntimeConfig,
11006    registry: &Registry,
11007    process_liveness: &SupervisorProcessLiveness,
11008    snapshot: &SharedSnapshot,
11009    _child: &mut Option<SupervisedChild>,
11010    failure: ReloadRegistrationFailure,
11011) -> Result<(), SuperviseError> {
11012    let ReloadRegistrationFailure {
11013        exit_report,
11014        reason,
11015    } = failure;
11016    match on_child_exit(
11017        spec,
11018        runtime.restart_policy,
11019        registry,
11020        snapshot,
11021        &runtime.terminal_ring,
11022        &runtime.spawn_events,
11023        &runtime.child_roster,
11024        exit_report,
11025        runtime.recovery_is_daemon,
11026    )
11027    .await
11028    {
11029        NextAction::Stop {
11030            registration_released,
11031        } => {
11032            if registration_released {
11033                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11034            }
11035        }
11036        NextAction::Restart { schedule } => {
11037            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11038                schedule.delay
11039            });
11040            if let Some(schedule) = schedule {
11041                log_crash_respawn(&spec.module_id, schedule);
11042            }
11043            schedule_respawn(
11044                runtime,
11045                snapshot,
11046                &spec.module_id,
11047                delay,
11048                RespawnKind::Spawn,
11049                true,
11050            )?;
11051        }
11052    }
11053    Err(SuperviseError::ReloadFailed {
11054        module_id: spec.module_id.clone(),
11055        reason,
11056    })
11057}
11058
11059async fn handle_reload_spawn_failure(
11060    spec: &ModuleSpec,
11061    runtime: &SupervisorRuntimeConfig,
11062    process_liveness: &SupervisorProcessLiveness,
11063    snapshot: &SharedSnapshot,
11064    _child: &mut Option<SupervisedChild>,
11065    reason: String,
11066) -> Result<(), SuperviseError> {
11067    let now = Instant::now();
11068    let mut schedule = None;
11069    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11070        clear_current_process_facts(state);
11071        if state.enabled {
11072            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11073            state.state = if schedule.is_some() {
11074                ModuleState::Restarting
11075            } else {
11076                if runtime
11077                    .failure_context(runtime.recovery_is_daemon)
11078                    .may_retry()
11079                {
11080                    state.failed_retry.schedule(&spec.module_id);
11081                }
11082                ModuleState::Failed
11083            };
11084        } else {
11085            state.state = ModuleState::Disabled;
11086        }
11087    })?;
11088    if let Some(schedule) = schedule {
11089        schedule_respawn(
11090            runtime,
11091            snapshot,
11092            &spec.module_id,
11093            schedule.delay,
11094            RespawnKind::Spawn,
11095            true,
11096        )?;
11097    } else {
11098        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11099    }
11100    Err(SuperviseError::ReloadFailed {
11101        module_id: spec.module_id.clone(),
11102        reason,
11103    })
11104}
11105
11106fn control_flags() -> Flags {
11107    Flags::new(false, Priority::Passive, false)
11108}
11109
11110#[allow(clippy::too_many_arguments)]
11111async fn drain_optional_child(
11112    module_id: &str,
11113    protocol: ModuleProtocol,
11114    stop_notice: StopNotice,
11115    registry: &Registry,
11116    forwarding: Option<&ForwardingTable>,
11117    snapshot: &SharedSnapshot,
11118    terminal_ring: &Arc<Mutex<TerminalRing>>,
11119    spawn_events: &SpawnEventFeed,
11120    child: &mut Option<SupervisedChild>,
11121    drain_timeout: Duration,
11122    final_state: ModuleState,
11123    enabled: Option<bool>,
11124    failure: &FailureContext,
11125) -> Result<(), SuperviseError> {
11126    if let Some(child) = child.take() {
11127        drain_child_to_state(
11128            module_id,
11129            protocol,
11130            stop_notice,
11131            registry,
11132            forwarding,
11133            snapshot,
11134            terminal_ring,
11135            spawn_events,
11136            child,
11137            drain_timeout,
11138            final_state,
11139            enabled,
11140            failure,
11141        )
11142        .await
11143    } else {
11144        update_snapshot(snapshot, Some(module_id), |state| {
11145            state.state = final_state;
11146            if final_state == ModuleState::Failed {
11147                state.failed_retry.failed_at = Some(Instant::now());
11148            }
11149            if let Some(enabled) = enabled {
11150                state.enabled = enabled;
11151            }
11152            clear_current_process_facts(state);
11153        })?;
11154        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11155    }
11156}
11157
11158#[allow(clippy::too_many_arguments)]
11159async fn drain_child_to_state(
11160    module_id: &str,
11161    _protocol: ModuleProtocol,
11162    stop_notice: StopNotice,
11163    registry: &Registry,
11164    forwarding: Option<&ForwardingTable>,
11165    snapshot: &SharedSnapshot,
11166    terminal_ring: &Arc<Mutex<TerminalRing>>,
11167    spawn_events: &SpawnEventFeed,
11168    mut child: SupervisedChild,
11169    drain_timeout: Duration,
11170    final_state: ModuleState,
11171    enabled: Option<bool>,
11172    failure: &FailureContext,
11173) -> Result<(), SuperviseError> {
11174    let protocol = child.protocol;
11175    update_snapshot(snapshot, Some(module_id), |state| {
11176        state.state = ModuleState::Draining;
11177        state.draining_to_replace = final_state == ModuleState::Restarting;
11178        if let Some(enabled) = enabled {
11179            state.enabled = enabled;
11180        }
11181    })?;
11182
11183    // The wait below is the same budget in every case; what differs is
11184    // whether anything has ASKED the child to stop before it starts. Only a
11185    // forwarding drain that reached the module's registered connection has
11186    // (`module.draining`, then a module GOODBYE). Every other child was told
11187    // nothing: a `protocol: "none"` module, which never registers; a subc
11188    // module spawned moments ago that has not sent HELLO yet; or a stop that
11189    // runs no forwarding drain. Without a signal the budget is only a delay
11190    // in front of SIGKILL -- and the not-yet-registered child is the worst
11191    // case, because it registers into a module that is already draining,
11192    // is never told, and is killed while healthy.
11193    if stop_notice != StopNotice::SentOverConnection {
11194        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11195            info!(
11196                module_id,
11197                pid = child.pid,
11198                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11199                "module has no connection yet; requesting stop by signal"
11200            );
11201        }
11202        request_graceful_stop(module_id, &child);
11203    }
11204
11205    let exit_report = match timeout(drain_timeout, child.wait()).await {
11206        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11207        Ok(Err(source)) => {
11208            fail_after_wait_error(
11209                module_id,
11210                snapshot,
11211                terminal_ring,
11212                spawn_events,
11213                format!("failed to wait during drain: {source}"),
11214                failure,
11215            );
11216            return Err(SuperviseError::Wait {
11217                module_id: module_id.to_string(),
11218                source,
11219            });
11220        }
11221        Err(_) => {
11222            // Mirror the sibling arm above: state is already `Draining`, and an
11223            // error propagated from here would strand it there -- a state
11224            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11225            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11226            // `Failed` before `?` keeps the module operator-visible and
11227            // revivable. Trigger is an ESRCH race (process exits between the
11228            // drain timeout firing and the kill) or a post-kill wait failure
11229            // (issue #34).
11230            //
11231            // Logged because the kill is otherwise visible only as signal 9 in
11232            // the terminal ring, and the budget it follows can be long enough
11233            // that consumers see a stretch of refusals with no stated cause.
11234            warn!(
11235                module_id,
11236                pid = child.pid,
11237                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11238                reason = ?final_state,
11239                ?stop_notice,
11240                "drain budget expired before the module exited; killing it"
11241            );
11242            child.start_kill().map_err(|source| {
11243                fail_after_wait_error(
11244                    module_id,
11245                    snapshot,
11246                    terminal_ring,
11247                    spawn_events,
11248                    format!("failed to kill during drain: {source}"),
11249                    failure,
11250                );
11251                SuperviseError::Kill {
11252                    module_id: module_id.to_string(),
11253                    source,
11254                }
11255            })?;
11256            let status = child.wait().await.map_err(|source| {
11257                fail_after_wait_error(
11258                    module_id,
11259                    snapshot,
11260                    terminal_ring,
11261                    spawn_events,
11262                    format!("failed to wait after drain kill: {source}"),
11263                    failure,
11264                );
11265                SuperviseError::Wait {
11266                    module_id: module_id.to_string(),
11267                    source,
11268                }
11269            })?;
11270            classify_reaped_child_exit(snapshot, &child, &status)
11271        }
11272    };
11273
11274    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11275    publish_after_terminal_record(
11276        module_id,
11277        || {
11278            record_terminal_with_detail(
11279                module_id,
11280                terminal_ring,
11281                spawn_events,
11282                &exit_report,
11283                terminal_disposition(final_state),
11284                detail,
11285            )
11286        },
11287        || {
11288            update_snapshot(snapshot, Some(module_id), |state| {
11289                state.state = final_state;
11290                if final_state == ModuleState::Failed {
11291                    state.failed_retry.failed_at = Some(Instant::now());
11292                }
11293                if let Some(enabled) = enabled {
11294                    state.enabled = enabled;
11295                }
11296                clear_current_process_facts(state);
11297                state.last_exit = Some(exit_report.clone());
11298                if exit_report.kind == ExitKind::DeliberateSeverance {
11299                    state.lifetime_restarts += 1;
11300                }
11301            })
11302        },
11303    )?;
11304    child.drain_stderr(module_id).await;
11305
11306    release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11307}
11308
11309/// Ask a child that nothing else has asked to stop, by signal.
11310///
11311/// A registered subc module is asked over its own connection: the drain sends
11312/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11313/// module GOODBYE, and the module stops itself. A module that speaks no subc
11314/// wire receives none of that, and neither does a subc module that has not
11315/// registered yet, so for them the drain budget would be pure delay in front of
11316/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11317/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11318/// into a recovery on the next start.
11319///
11320/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
11321/// rule rather than an optimisation: that module's graceful stop is already
11322/// running by the time its child is drained, and a signal would race it.
11323///
11324/// Best-effort by construction. A child that has already exited is the ordinary
11325/// case rather than an error (the kill lands on a reaped or exiting pid), so a
11326/// failure is logged at debug and the wait-then-kill below still decides the
11327/// outcome.
11328#[cfg(unix)]
11329fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11330    let Some(pid) = child
11331        .id()
11332        .and_then(|pid| i32::try_from(pid).ok())
11333        .and_then(rustix::process::Pid::from_raw)
11334    else {
11335        debug!(
11336            module_id,
11337            "no pid to signal for teardown; falling through to the drain wait"
11338        );
11339        return;
11340    };
11341    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11342        Ok(()) => debug!(
11343            module_id,
11344            "sent SIGTERM to a module nothing else asked to stop"
11345        ),
11346        Err(err) => debug!(
11347            module_id,
11348            error = %err,
11349            "SIGTERM to module failed; the drain wait and kill still apply"
11350        ),
11351    }
11352}
11353
11354/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11355/// Windows does offer need cooperation this supervisor cannot assume: a console
11356/// control event requires sharing a console with the child, and `WM_CLOSE`
11357/// requires the child to pump a message loop. A supervised server process does
11358/// neither, so there is nothing to send and teardown is the wait followed by the
11359/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11360/// the thing `protocol: "none"` exists to avoid.
11361#[cfg(not(unix))]
11362fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11363    debug!(
11364        module_id,
11365        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11366    );
11367}
11368
11369fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11370    match final_state {
11371        ModuleState::Stopped => TerminalDisposition::Stopped,
11372        ModuleState::Disabled => TerminalDisposition::Disabled,
11373        ModuleState::Restarting => TerminalDisposition::Restarting,
11374        ModuleState::Failed => TerminalDisposition::Failed,
11375        ModuleState::Starting
11376        | ModuleState::Running
11377        | ModuleState::Unresponsive
11378        | ModuleState::Draining => {
11379            unreachable!("terminal exits only finish in terminal or restarting states")
11380        }
11381    }
11382}
11383
11384/// Release a reaped child's registration before allowing another spawn.
11385///
11386/// EOF is not a process-lifetime signal: an inherited socket can stay open
11387/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11388/// reading EOF. After the normal release grace, request connection close (which
11389/// cancels both reads and dispatch), then allow one more release grace for the
11390/// connection guard's forwarding cleanup. Never evict a different connection.
11391async fn release_dead_registration(
11392    registry: &Registry,
11393    forwarding: Option<&ForwardingTable>,
11394    snapshot: &SharedSnapshot,
11395    module_id: &str,
11396    failure: &FailureContext,
11397) -> Result<(), SuperviseError> {
11398    let result = async {
11399        let registration = registry
11400            .get_module(module_id)
11401            .map_err(SuperviseError::Registry)?;
11402        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11403            Ok(()) => return Ok(()),
11404            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11405            Err(err) => return Err(err),
11406        }
11407        let pid = lock_snapshot(snapshot)?.reaped_pid;
11408        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11409            warn!(
11410                module_id,
11411                pid,
11412                connection_id = registration.connection_id.get(),
11413                "reaped module registration outlived release grace; closing dead connection"
11414            );
11415            forwarding.request_connection_close(
11416                registration.connection_id,
11417                CloseReason::new(
11418                    "supervised_process_reaped",
11419                    format!("module '{module_id}' pid {pid} exited"),
11420                ),
11421            );
11422            wait_for_slot_registration_release(
11423                registry,
11424                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11425                REGISTRY_RELEASE_TIMEOUT,
11426            )
11427            .await?;
11428        }
11429        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11430    }
11431    .await;
11432    if let Err(err) = &result {
11433        fail_snapshot(snapshot, Some(module_id), None, failure);
11434        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11435    }
11436    result
11437}
11438
11439/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11440/// plain stop or restart waits for before it spawns a replacement.
11441async fn wait_for_registration_release(
11442    registry: &Registry,
11443    module_id: &str,
11444    wait: Duration,
11445) -> Result<(), SuperviseError> {
11446    wait_for_slot_registration_release(
11447        registry,
11448        crate::registry::RegistrationSlot::Active(module_id),
11449        wait,
11450    )
11451    .await
11452}
11453
11454/// Wait for the registration in `slot` to go away.
11455///
11456/// Keyed on the slot rather than the bare module id because a successful swap
11457/// never empties the id's active slot (the promoted candidate is in it), so an
11458/// id-keyed wait for the incumbent's release would always time out. Draining a
11459/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11460/// incumbent's connection instead.
11461async fn wait_for_slot_registration_release(
11462    registry: &Registry,
11463    slot: crate::registry::RegistrationSlot<'_>,
11464    wait: Duration,
11465) -> Result<(), SuperviseError> {
11466    let deadline = Instant::now() + wait;
11467    let mut release_events = registration_release_events().subscribe();
11468    let still_active = |registration: &crate::registry::ModuleRegistration| {
11469        SuperviseError::RegistrationStillActive {
11470            module_id: registration.manifest.module_id.clone(),
11471            waited: wait,
11472        }
11473    };
11474    loop {
11475        let _observed_generation = *release_events.borrow_and_update();
11476        let Some(registration) = registry
11477            .registration(slot)
11478            .map_err(SuperviseError::Registry)?
11479        else {
11480            return Ok(());
11481        };
11482
11483        let now = Instant::now();
11484        if now >= deadline {
11485            return Err(still_active(&registration));
11486        }
11487
11488        let remaining = deadline.saturating_duration_since(now);
11489        match timeout(remaining, release_events.changed()).await {
11490            Ok(Ok(())) | Ok(Err(_)) => {}
11491            Err(_) => return Err(still_active(&registration)),
11492        }
11493    }
11494}
11495
11496#[cfg(test)]
11497mod slot_registration_wait_tests {
11498    use super::*;
11499    use crate::registry::{ConnectionId, RegistrationSlot};
11500    use subc_protocol::manifest::ModuleManifest;
11501
11502    #[tokio::test]
11503    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11504        let registry = Arc::new(Registry::default());
11505        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11506        let runtime = supervisor.runtime_config();
11507        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11508        let spec = ModuleSpec {
11509            module_id: "enable-stale-registration".to_string(),
11510            program: PathBuf::from("/missing/enable-retry-test"),
11511            args: Vec::new(),
11512            env: Vec::new(),
11513            reserved: false,
11514            reserved_prefixes: Vec::new(),
11515            protocol: ModuleProtocol::Subc,
11516            overlap: Default::default(),
11517        };
11518        let connection = ConnectionId::new(90);
11519        registry
11520            .register_with_control_ops(
11521                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11522                1,
11523                connection,
11524                Vec::new(),
11525            )
11526            .unwrap();
11527        let mut child = None;
11528        let err = set_child_enabled(
11529            &spec,
11530            &runtime,
11531            &registry,
11532            &supervisor.process_liveness,
11533            &snapshot,
11534            &mut child,
11535            true,
11536            &runtime.failure_context(false),
11537        )
11538        .await
11539        .unwrap_err();
11540        assert!(matches!(
11541            err,
11542            SuperviseError::RegistrationStillActive { .. }
11543        ));
11544        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11545        assert!(child.is_none());
11546        registry.deregister_connection(connection).unwrap();
11547        let err = set_child_enabled(
11548            &spec,
11549            &runtime,
11550            &registry,
11551            &supervisor.process_liveness,
11552            &snapshot,
11553            &mut child,
11554            true,
11555            &runtime.failure_context(false),
11556        )
11557        .await
11558        .unwrap_err();
11559        assert!(
11560            matches!(err, SuperviseError::Spawn { .. }),
11561            "second enable must attempt a spawn: {err}"
11562        );
11563        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11564    }
11565
11566    const INCUMBENT: u64 = 1;
11567    const CANDIDATE: u64 = 2;
11568
11569    fn swapped_registry() -> Arc<Registry> {
11570        let registry = Arc::new(Registry::default());
11571        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11572        registry
11573            .register_with_control_ops(
11574                manifest.clone(),
11575                1,
11576                ConnectionId::new(INCUMBENT),
11577                Vec::new(),
11578            )
11579            .unwrap();
11580        registry
11581            .register_candidate_with_control_ops(
11582                manifest,
11583                1,
11584                ConnectionId::new(CANDIDATE),
11585                Vec::new(),
11586            )
11587            .unwrap();
11588        registry
11589    }
11590
11591    /// After a promotion the id's active slot is held by the new process, so an
11592    /// id-keyed wait for the incumbent's release can never succeed; the
11593    /// connection-keyed wait completes as soon as the incumbent deregisters.
11594    #[tokio::test]
11595    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11596        let registry = swapped_registry();
11597        registry.promote_candidate("m").unwrap().unwrap();
11598
11599        assert!(matches!(
11600            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11601            Err(SuperviseError::RegistrationStillActive { .. })
11602        ));
11603
11604        // Still held while the incumbent's connection has not deregistered.
11605        assert!(matches!(
11606            wait_for_slot_registration_release(
11607                &registry,
11608                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11609                Duration::from_millis(50),
11610            )
11611            .await,
11612            Err(SuperviseError::RegistrationStillActive { .. })
11613        ));
11614
11615        let releaser = Arc::clone(&registry);
11616        let release = tokio::spawn(async move {
11617            sleep(Duration::from_millis(20)).await;
11618            releaser
11619                .deregister_connection(ConnectionId::new(INCUMBENT))
11620                .unwrap();
11621            notify_registration_release();
11622        });
11623        wait_for_slot_registration_release(
11624            &registry,
11625            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11626            Duration::from_secs(5),
11627        )
11628        .await
11629        .expect("the incumbent's own registration is released");
11630        release.await.unwrap();
11631        assert!(registry.get_module("m").unwrap().is_some());
11632    }
11633
11634    /// The candidate slot is waited on separately from the active slot: the
11635    /// incumbent's registration neither holds up nor stands in for it.
11636    #[tokio::test]
11637    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11638        let registry = swapped_registry();
11639        assert!(matches!(
11640            wait_for_slot_registration_release(
11641                &registry,
11642                RegistrationSlot::Candidate("m"),
11643                Duration::from_millis(50),
11644            )
11645            .await,
11646            Err(SuperviseError::RegistrationStillActive { .. })
11647        ));
11648        registry
11649            .deregister_connection(ConnectionId::new(CANDIDATE))
11650            .unwrap();
11651        wait_for_slot_registration_release(
11652            &registry,
11653            RegistrationSlot::Candidate("m"),
11654            Duration::from_millis(50),
11655        )
11656        .await
11657        .expect("a candidate slot with no candidate is released");
11658        assert!(registry
11659            .registration(RegistrationSlot::Active("m"))
11660            .unwrap()
11661            .is_some());
11662    }
11663}
11664
11665fn classify_exit(status: &ExitStatus) -> ExitReport {
11666    ExitReport {
11667        kind: if status.success() {
11668            ExitKind::Clean
11669        } else {
11670            ExitKind::Crash
11671        },
11672        code: status.code(),
11673        signal: exit_signal(status),
11674        at_ms: unix_ms_now(),
11675    }
11676}
11677
11678/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11679/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11680/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11681/// disposition still must be `Failed` so the terminal ring is not silently missing
11682/// an entry, matching what `fail_snapshot` records for this same arm.
11683fn wait_error_exit_report() -> ExitReport {
11684    ExitReport {
11685        kind: ExitKind::Crash,
11686        code: None,
11687        signal: None,
11688        at_ms: unix_ms_now(),
11689    }
11690}
11691
11692#[cfg(unix)]
11693fn exit_signal(status: &ExitStatus) -> Option<i32> {
11694    use std::os::unix::process::ExitStatusExt;
11695
11696    status.signal()
11697}
11698
11699#[cfg(not(unix))]
11700fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11701    None
11702}
11703
11704/// Give an operator-touched module its full crash budget back.
11705///
11706/// Named for the counter it used to zero; it now empties the in-window ring,
11707/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11708/// ledger of what happened survives every operator action.
11709fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11710    update_snapshot(snapshot, Some(module_id), |state| {
11711        state.clear_crash_restarts();
11712    })
11713}
11714
11715fn set_running(
11716    snapshot: &SharedSnapshot,
11717    child: &SupervisedChild,
11718    module_id: &str,
11719    spawn_events: &SpawnEventFeed,
11720) -> Result<(), SuperviseError> {
11721    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11722        module_id: Some(module_id.to_string()),
11723    })?;
11724    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11725    if std::mem::take(&mut state.coalesced_restart_pending) {
11726        let generation = state.spawn_generation;
11727        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11728    }
11729    state.drain_disposition_detail = None;
11730    state.spawn_failure = None;
11731    state.waiting_for_previous_pid = None;
11732    state.waiting_for_previous_detail = None;
11733    // Every caller of this is a plain spawn, which always uses the primary key;
11734    // a promoted swap candidate sets the flag itself after this returns.
11735    state.in_alternate_slot = false;
11736    state.configuration_updated_since_spawn = false;
11737    state.spawned_protocol = Some(child.protocol);
11738    state.state = ModuleState::Running;
11739    state.enabled = true;
11740    state.process_alive = true;
11741    state.pid = child.id();
11742    // Probe samples belong to one process. Keep module-level escalation and
11743    // late-answer history, but never charge a new child for its predecessor's misses.
11744    state.health.status = SupervisorHealthStatus::Unknown;
11745    state.health.consecutive_failures = 0;
11746    state.health.detail = None;
11747    state.health.metrics = None;
11748    state.health.last_probe_ms = None;
11749    state.failed_retry.healthy_since = None;
11750    state.failed_retry.failed_at = None;
11751    state.failed_retry.running_since = Some(child.spawned_at);
11752    #[cfg(windows)]
11753    {
11754        state.pipe_consumed = child
11755            .nonce_delivery
11756            .as_ref()
11757            .map(|delivery| delivery.consumed());
11758    }
11759    #[cfg(target_os = "macos")]
11760    {
11761        state.report_ready = Some(Arc::clone(&child.report_ready));
11762    }
11763    state.spawned_at_ms = Some(child.spawned_at_ms);
11764    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11765    state.spawned_from = Some(child.spawned_from.clone());
11766    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11767    state.spawned_file_identity = child.spawned_file_identity.clone();
11768    state.process_start_time = child.process_start_time;
11769    Ok(())
11770}
11771
11772fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11773    state.failed_retry.healthy_since = None;
11774    state.failed_retry.running_since = None;
11775    state.waiting_for_previous_pid = None;
11776    state.waiting_for_previous_detail = None;
11777    state.process_alive = false;
11778    state.spawned_protocol = None;
11779    state.pid = None;
11780    #[cfg(windows)]
11781    {
11782        state.pipe_consumed = None;
11783    }
11784    #[cfg(target_os = "macos")]
11785    {
11786        state.report_ready = None;
11787    }
11788    state.spawned_at_ms = None;
11789    state.exec_outcome = None;
11790    state.spawned_from = None;
11791    state.launch_spec_digest = None;
11792    state.spawned_file_identity = None;
11793    state.process_start_time = None;
11794    state.deliberate_severance = None;
11795}
11796
11797#[cfg(test)]
11798fn record_deliberate_severance(
11799    snapshot: &SharedSnapshot,
11800    identity: ProcessIdentity,
11801) -> Result<(), SuperviseError> {
11802    update_snapshot(snapshot, None, |state| {
11803        state.deliberate_severance = Some(identity);
11804    })
11805}
11806
11807fn apply_deliberate_severance_marker(
11808    snapshot: &SharedSnapshot,
11809    exited_identity: Option<ProcessIdentity>,
11810    mut exit_report: ExitReport,
11811) -> ExitReport {
11812    let marker = lock_snapshot(snapshot)
11813        .ok()
11814        .and_then(|mut state| state.deliberate_severance.take());
11815    if marker.is_some() && marker == exited_identity {
11816        exit_report.kind = ExitKind::DeliberateSeverance;
11817    }
11818    exit_report
11819}
11820
11821fn classify_reaped_child_exit(
11822    snapshot: &SharedSnapshot,
11823    child: &SupervisedChild,
11824    status: &ExitStatus,
11825) -> ExitReport {
11826    let _ = update_snapshot(snapshot, None, |state| {
11827        state.reaped_pid = Some(child.pid);
11828        state.spawn_failure = child.spawn_failure.clone();
11829    });
11830    if child.spawn_failure.as_deref() == Some("registration_timeout") {
11831        registration_failure_exit_report(classify_exit(status))
11832    } else {
11833        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11834    }
11835}
11836
11837fn fail_snapshot(
11838    snapshot: &SharedSnapshot,
11839    module_id: Option<&str>,
11840    last_exit: Option<ExitReport>,
11841    failure: &FailureContext,
11842) {
11843    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11844        state.state = ModuleState::Failed;
11845        clear_current_process_facts(state);
11846        if failure.may_retry() && state.enabled {
11847            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
11848        } else if failure.shutdown.is_set() {
11849            state.failed_retry.cancel();
11850        }
11851        if let Some(last_exit) = last_exit {
11852            state.last_exit = Some(last_exit);
11853        }
11854    }) {
11855        error!(error = %err, "failed to mark supervisor state failed");
11856    }
11857}
11858
11859fn update_snapshot(
11860    snapshot: &SharedSnapshot,
11861    module_id: Option<&str>,
11862    update: impl FnOnce(&mut SupervisorSnapshot),
11863) -> Result<(), SuperviseError> {
11864    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11865        module_id: module_id.map(ToOwned::to_owned),
11866    })?;
11867    update(&mut state);
11868    Ok(())
11869}
11870
11871const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11872
11873fn lock_snapshot_for_control<'a>(
11874    snapshot: &'a SharedSnapshot,
11875    module_id: &str,
11876    caller: &'static str,
11877) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11878    let started_at = Instant::now();
11879    let guard = lock_snapshot(snapshot)?;
11880    let waited = started_at.elapsed();
11881    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11882        warn!(
11883            module_id = %module_id,
11884            waited_ms = waited.as_millis() as u64,
11885            caller = %caller,
11886            "slow snapshot lock"
11887        );
11888    }
11889    Ok(guard)
11890}
11891
11892fn lock_snapshot(
11893    snapshot: &SharedSnapshot,
11894) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11895    snapshot
11896        .lock()
11897        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11898}
11899
11900#[cfg(test)]
11901mod terminal_history_tests {
11902    use std::{
11903        path::PathBuf,
11904        sync::Arc,
11905        time::{Duration, Instant},
11906    };
11907
11908    use tokio::{sync::mpsc, time::sleep};
11909
11910    use super::{
11911        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11912        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11913        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11914        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11915        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11916        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11917        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11918        SupervisorHealthStatus, SupervisorSnapshot,
11919    };
11920    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11921    // use for their own wall-clock deadlines: crash-restart instants must be on
11922    // the same clock the production code stamps them with, which is tokio's (and
11923    // is what `start_paused` tests can move).
11924    use super::Instant as ClockInstant;
11925    use crate::{
11926        registry::Registry,
11927        terminal_ring::{TerminalRing, TerminalRingConfig},
11928    };
11929    use std::sync::Mutex;
11930    use subc_control::TerminalDisposition;
11931
11932    /// See the twin in `control.rs` for why this derives the path from
11933    /// `current_exe()` and why the existence check is here: `--lib` alone does
11934    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11935    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11936    pub(super) fn fake_aft_stub_path() -> PathBuf {
11937        let mut path = std::env::current_exe().expect("current_exe available in tests");
11938        path.pop();
11939        path.pop();
11940        path.push(if cfg!(windows) {
11941            "fake-aft-stub.exe"
11942        } else {
11943            "fake-aft-stub"
11944        });
11945        assert!(
11946            path.exists(),
11947            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11948             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11949            path.display()
11950        );
11951        path
11952    }
11953
11954    #[test]
11955    fn reserved_never_spawned_refuses_every_hello() {
11956        // The canary hole: a reserved id whose module has never spawned had NO
11957        // gate entry and admitted anyone -- the reservation protected the nonce
11958        // holder, not the NAME. Now the entry is present with no legitimate
11959        // holder and refuses all comers.
11960        let supervisor = SupervisorHandle::default();
11961        supervisor.apply_identity_configuration(&ModuleSpec {
11962            module_id: "never-spawned".to_string(),
11963            program: PathBuf::from("/usr/bin/false"),
11964            args: Vec::new(),
11965            env: Vec::new(),
11966            reserved: true,
11967            reserved_prefixes: Vec::new(),
11968            protocol: ModuleProtocol::Subc,
11969            overlap: Default::default(),
11970        });
11971        assert!(
11972            supervisor
11973                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11974                .is_err(),
11975            "forged nonce must refuse on a reserved never-spawned id"
11976        );
11977        assert!(
11978            supervisor
11979                .reserved_hello_verification("never-spawned", None)
11980                .is_err(),
11981            "absent nonce must refuse on a reserved never-spawned id"
11982        );
11983        // And a real spawn nonce minted later admits exactly that nonce.
11984        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11985        supervisor.apply_identity_configuration(&ModuleSpec {
11986            module_id: "never-spawned".to_string(),
11987            program: PathBuf::from("/usr/bin/false"),
11988            args: Vec::new(),
11989            env: Vec::new(),
11990            reserved: true,
11991            reserved_prefixes: Vec::new(),
11992            protocol: ModuleProtocol::Subc,
11993            overlap: Default::default(),
11994        });
11995        assert!(supervisor
11996            .reserved_hello_verification("never-spawned", Some("minted"))
11997            .is_ok());
11998        assert!(supervisor
11999            .reserved_hello_verification("never-spawned", Some("forged"))
12000            .is_err());
12001    }
12002
12003    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12004    /// happened, which is what "spent budget" looks like to every reader.
12005    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12006        let now = ClockInstant::now();
12007        for _ in 0..count {
12008            state.crash_restarts.push_back(now);
12009        }
12010    }
12011
12012    /// Age the oldest recorded restart out of `window`, standing in for the hours
12013    /// that would otherwise have to pass. Injecting the instant is the point: a
12014    /// test that slept a real window would take ten minutes and still prove less.
12015    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12016        let aged = state
12017            .crash_restarts
12018            .front()
12019            .expect("a crash restart must be recorded before it can be aged")
12020            .checked_sub(window + Duration::from_secs(1))
12021            .expect("the test clock is far enough from its origin to age an instant");
12022        state.crash_restarts[0] = aged;
12023    }
12024
12025    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12026        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12027        seed_crash_restarts(&mut state, count);
12028        state
12029    }
12030
12031    #[test]
12032    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12033        let policy = RestartPolicy::new(3, Duration::ZERO);
12034        let now = ClockInstant::now();
12035        assert!(daemon_will_restart(
12036            &mut snapshot_with_restarts(true, 2),
12037            &policy,
12038            now
12039        ));
12040        assert!(!daemon_will_restart(
12041            &mut snapshot_with_restarts(true, 3),
12042            &policy,
12043            now
12044        ));
12045        assert!(!daemon_will_restart(
12046            &mut snapshot_with_restarts(false, 0),
12047            &policy,
12048            now
12049        ));
12050    }
12051
12052    #[test]
12053    fn crash_restart_backoff_escalates_with_in_window_count() {
12054        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12055            .with_max_backoff(Duration::from_secs(30));
12056        let now = ClockInstant::now();
12057        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12058        let schedules = (0..4)
12059            .map(|_| {
12060                state
12061                    .next_crash_restart(&policy, now)
12062                    .expect("the test policy allows four crash restarts")
12063            })
12064            .collect::<Vec<_>>();
12065
12066        assert_eq!(
12067            schedules
12068                .iter()
12069                .map(|schedule| schedule.restart_in_window)
12070                .collect::<Vec<_>>(),
12071            vec![0, 1, 2, 3]
12072        );
12073        assert_eq!(
12074            schedules
12075                .iter()
12076                .map(|schedule| schedule.delay)
12077                .collect::<Vec<_>>(),
12078            vec![
12079                Duration::from_millis(100),
12080                Duration::from_secs(1),
12081                Duration::from_secs(10),
12082                Duration::from_secs(30),
12083            ]
12084        );
12085    }
12086
12087    #[test]
12088    fn crash_restart_backoff_resets_after_ring_clear() {
12089        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12090        let now = ClockInstant::now();
12091        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12092        assert_eq!(
12093            state.next_crash_restart(&policy, now).unwrap().delay,
12094            Duration::from_millis(100)
12095        );
12096        assert_eq!(
12097            state.next_crash_restart(&policy, now).unwrap().delay,
12098            Duration::from_secs(1)
12099        );
12100
12101        state.clear_crash_restarts();
12102        let schedule = state
12103            .next_crash_restart(&policy, now)
12104            .expect("a cleared ring must allow another restart");
12105        assert_eq!(schedule.restart_in_window, 0);
12106        assert_eq!(schedule.delay, Duration::from_millis(100));
12107    }
12108
12109    #[test]
12110    fn crash_restart_backoff_ignores_aged_restarts() {
12111        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12112        let now = ClockInstant::now();
12113        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12114        state
12115            .next_crash_restart(&policy, now)
12116            .expect("the first restart is allowed");
12117        state
12118            .next_crash_restart(&policy, now)
12119            .expect("the second restart is allowed");
12120        state.crash_restarts[0] = now
12121            .checked_sub(policy.window + Duration::from_secs(1))
12122            .expect("the fake clock can age a restart past the window");
12123
12124        let schedule = state
12125            .next_crash_restart(&policy, now)
12126            .expect("an aged restart must release its slot");
12127        assert_eq!(schedule.restart_in_window, 1);
12128        assert_eq!(schedule.delay, Duration::from_secs(1));
12129        assert_eq!(state.crash_restarts.len(), 2);
12130    }
12131
12132    /// The budget is a rate: the same three spent restarts refuse a respawn
12133    /// while they are recent and allow one once they have aged past the window.
12134    /// Nothing about the module changed in between, which is the whole point.
12135    #[test]
12136    fn a_budget_spent_before_the_window_no_longer_refuses() {
12137        let policy = RestartPolicy::new(3, Duration::ZERO);
12138        let mut state = snapshot_with_restarts(true, 3);
12139        let now = ClockInstant::now();
12140        assert!(!daemon_will_restart(&mut state, &policy, now));
12141
12142        assert!(daemon_will_restart(
12143            &mut state,
12144            &policy,
12145            now + policy.window + Duration::from_secs(1)
12146        ));
12147        assert!(
12148            state.crash_restarts.is_empty(),
12149            "reading the budget must drop the instants that left the window"
12150        );
12151    }
12152
12153    fn module_with_recovery_snapshot(
12154        state: ModuleState,
12155        enabled: bool,
12156        restart_count: u32,
12157    ) -> SupervisedModule {
12158        let registry = Arc::new(Registry::default());
12159        let supervisor =
12160            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12161        let runtime = supervisor.runtime_config();
12162        let spec = ModuleSpec {
12163            module_id: "recovery-snapshot".to_string(),
12164            program: fake_aft_stub_path(),
12165            args: Vec::new(),
12166            env: Vec::new(),
12167            reserved: false,
12168            reserved_prefixes: Vec::new(),
12169            protocol: ModuleProtocol::Subc,
12170            overlap: Default::default(),
12171        };
12172        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12173        seed_crash_restarts(&mut snapshot, restart_count);
12174        // These tests read synthetic snapshots. A real child and monitor would
12175        // race those reads by replacing the requested state during startup.
12176        let (commands, _rx) = mpsc::channel(4);
12177        SupervisedModule {
12178            inner: Arc::new(SupervisedModuleInner {
12179                module_id: spec.module_id.clone(),
12180                registry,
12181                snapshot: Arc::new(Mutex::new(snapshot)),
12182                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12183                    spec,
12184                    health: runtime.health,
12185                })),
12186                stderr_ring: runtime.stderr_ring,
12187                terminal_ring: runtime.terminal_ring,
12188                commands,
12189                monitor: Mutex::new(None),
12190                restart_policy: runtime.restart_policy,
12191                effective_drain_timeout: runtime.effective_drain_timeout,
12192                provenance_probe: supervisor.provenance_probe.clone(),
12193            }),
12194        }
12195    }
12196
12197    #[cfg(target_os = "linux")]
12198    #[tokio::test]
12199    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12200        let supervisor =
12201            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12202                .with_cgroup_placement(None);
12203        let result = supervisor.spawn(ModuleSpec {
12204            module_id: "no-cgroup-placement".to_string(),
12205            program: fake_aft_stub_path(),
12206            args: Vec::new(),
12207            env: Vec::new(),
12208            reserved: false,
12209            reserved_prefixes: Vec::new(),
12210            protocol: ModuleProtocol::Subc,
12211            overlap: Default::default(),
12212        });
12213
12214        assert!(
12215            result.is_ok(),
12216            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12217        );
12218    }
12219
12220    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12221    async fn undecided_snapshot_uses_shared_restart_predicate() {
12222        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12223            .will_recover_after_connection_loss()
12224            .unwrap());
12225        assert!(
12226            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12227                .will_recover_after_connection_loss()
12228                .unwrap()
12229        );
12230    }
12231
12232    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12233    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12234        assert!(
12235            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12236                .will_recover_after_connection_loss()
12237                .unwrap()
12238        );
12239    }
12240
12241    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12242    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12243        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12244            .will_recover_after_connection_loss()
12245            .unwrap());
12246        assert!(
12247            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12248                .will_recover_after_connection_loss()
12249                .unwrap()
12250        );
12251    }
12252
12253    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12254    async fn warming_snapshot_is_limited_to_startup_phases() {
12255        for state in [
12256            ModuleState::Starting,
12257            ModuleState::Running,
12258            ModuleState::Restarting,
12259        ] {
12260            assert!(
12261                module_with_recovery_snapshot(state, true, 0)
12262                    .is_warming()
12263                    .unwrap(),
12264                "{state:?} should be warming"
12265            );
12266        }
12267        for state in [
12268            ModuleState::Unresponsive,
12269            ModuleState::Draining,
12270            ModuleState::Stopped,
12271            ModuleState::Failed,
12272            ModuleState::Disabled,
12273        ] {
12274            assert!(
12275                !module_with_recovery_snapshot(state, true, 0)
12276                    .is_warming()
12277                    .unwrap(),
12278                "{state:?} should not be warming"
12279            );
12280        }
12281    }
12282
12283    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12284    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12285        let registry = Arc::new(Registry::default());
12286        let supervisor =
12287            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12288        let module = supervisor
12289            .spawn(ModuleSpec {
12290                module_id: "terminal-history".to_string(),
12291                program: fake_aft_stub_path(),
12292                args: Vec::new(),
12293                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12294                reserved: false,
12295                reserved_prefixes: Vec::new(),
12296                protocol: ModuleProtocol::Subc,
12297                overlap: Default::default(),
12298            })
12299            .unwrap();
12300
12301        let deadline = Instant::now() + Duration::from_secs(5);
12302        loop {
12303            let history = module.terminal_history();
12304            if history.entries.len() == 2 {
12305                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12306                assert_eq!(history.dropped, 0);
12307                assert_eq!(
12308                    history
12309                        .entries
12310                        .iter()
12311                        .map(|entry| entry.exit_code)
12312                        .collect::<Vec<_>>(),
12313                    vec![Some(23), Some(23)]
12314                );
12315                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12316                return;
12317            }
12318            assert!(
12319                Instant::now() < deadline,
12320                "module did not retain two terminal exits: {history:?}"
12321            );
12322            sleep(Duration::from_millis(10)).await;
12323        }
12324    }
12325
12326    /// A disable issued while a crash respawn is still backing off must preempt
12327    /// that respawn: the operator's stop wins, the disable must not queue behind
12328    /// the backoff, and the module must never come back up afterwards.
12329    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12330    async fn disable_during_crash_backoff_cancels_pending_respawn() {
12331        let backoff = Duration::from_secs(2);
12332        let supervisor = Supervisor::new_for_test(
12333            Arc::new(Registry::default()),
12334            RestartPolicy::new(10, backoff),
12335        );
12336        let module = supervisor
12337            .spawn(ModuleSpec {
12338                module_id: "disable-during-backoff".to_string(),
12339                program: fake_aft_stub_path(),
12340                args: Vec::new(),
12341                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12342                reserved: false,
12343                reserved_prefixes: Vec::new(),
12344                protocol: ModuleProtocol::Subc,
12345                overlap: Default::default(),
12346            })
12347            .unwrap();
12348
12349        // Wait for the first crash to put the module into its backoff window.
12350        let deadline = Instant::now() + Duration::from_secs(5);
12351        loop {
12352            if module.status().unwrap().state == ModuleState::Restarting {
12353                break;
12354            }
12355            assert!(
12356                Instant::now() < deadline,
12357                "module never entered the crash backoff"
12358            );
12359            sleep(Duration::from_millis(10)).await;
12360        }
12361
12362        let started = Instant::now();
12363        module.set_enabled(false).await.unwrap();
12364        let waited = started.elapsed();
12365
12366        assert!(
12367            waited < backoff / 2,
12368            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12369        );
12370        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12371
12372        // Outlast the backoff: the respawn it was counting down to must never run.
12373        sleep(backoff + Duration::from_millis(500)).await;
12374        let status = module.status().unwrap();
12375        assert_eq!(status.state, ModuleState::Disabled);
12376        assert_eq!(
12377            status.spawn_generation, 1,
12378            "module respawned after the operator disabled it"
12379        );
12380    }
12381
12382    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12383    /// the shape of nats-server, the program this rule exists for.
12384    #[cfg(unix)]
12385    fn protocol_none_sigterm_exits_clean_spec(
12386        module_id: &str,
12387        dir: &std::path::Path,
12388    ) -> (ModuleSpec, PathBuf, PathBuf) {
12389        let ready = dir.join("ready");
12390        let marker = dir.join("sigterm");
12391        let spec = ModuleSpec {
12392            module_id: module_id.to_string(),
12393            program: fake_aft_stub_path(),
12394            args: Vec::new(),
12395            env: vec![
12396                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12397                (
12398                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12399                    marker.display().to_string(),
12400                ),
12401                (
12402                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12403                    ready.display().to_string(),
12404                ),
12405            ],
12406            reserved: false,
12407            reserved_prefixes: Vec::new(),
12408            protocol: ModuleProtocol::None,
12409            overlap: Default::default(),
12410        };
12411        (spec, ready, marker)
12412    }
12413
12414    /// Wait for a file the child writes, so a signal is never sent before the
12415    /// child's SIGTERM handler is installed (the default disposition would
12416    /// kill it by signal and the exit would not be clean).
12417    #[cfg(unix)]
12418    async fn wait_for_file(path: &std::path::Path) {
12419        let deadline = Instant::now() + Duration::from_secs(10);
12420        while !path.exists() {
12421            assert!(
12422                Instant::now() < deadline,
12423                "{} never appeared",
12424                path.display()
12425            );
12426            sleep(Duration::from_millis(10)).await;
12427        }
12428    }
12429
12430    /// A protocol-none module that exits 0 because something OUTSIDE the
12431    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12432    /// the crash-path disposition rather than `stopped`.
12433    #[cfg(unix)]
12434    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12435    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12436        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12437        let (spec, ready, marker) =
12438            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12439        let supervisor = Supervisor::new_for_test(
12440            Arc::new(Registry::default()),
12441            RestartPolicy::new(3, Duration::ZERO),
12442        );
12443        let module = supervisor.spawn(spec).unwrap();
12444        wait_for_file(&ready).await;
12445        // The ready file proves the child installed its SIGTERM handler, not
12446        // that the supervisor has processed the privacy trampoline's exec
12447        // acknowledgement. On macOS status withholds the pid until then.
12448        let deadline = Instant::now() + Duration::from_secs(10);
12449        let first_pid = loop {
12450            let status = module.status().unwrap();
12451            if status.state == ModuleState::Running {
12452                if let Some(pid) = status.pid {
12453                    break pid;
12454                }
12455            }
12456            assert!(
12457                Instant::now() < deadline,
12458                "a running module must report its pid after exec confirmation: {status:?}"
12459            );
12460            sleep(Duration::from_millis(10)).await;
12461        };
12462
12463        rustix::process::kill_process(
12464            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12465            rustix::process::Signal::TERM,
12466        )
12467        .unwrap();
12468
12469        let deadline = Instant::now() + Duration::from_secs(10);
12470        let respawned = loop {
12471            let status = module.status().unwrap();
12472            if status.state == ModuleState::Running
12473                && status.pid.is_some_and(|pid| pid != first_pid)
12474            {
12475                break status;
12476            }
12477            assert!(
12478                Instant::now() < deadline,
12479                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12480            );
12481            sleep(Duration::from_millis(10)).await;
12482        };
12483        assert_eq!(respawned.spawn_generation, 2);
12484        assert!(
12485            marker.exists(),
12486            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12487        );
12488
12489        let history = module.terminal_history();
12490        assert_eq!(history.entries.len(), 1, "{history:?}");
12491        let entry = &history.entries[0];
12492        assert_eq!(entry.exit_code, Some(0));
12493        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12494        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12495
12496        module.stop().await.unwrap();
12497    }
12498
12499    /// Repeated unrequested clean exits of a protocol-none module spend the
12500    /// restart budget exactly as crashes do, and the module ends `failed` with
12501    /// the budget named.
12502    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12503    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12504        let supervisor = Supervisor::new_for_test(
12505            Arc::new(Registry::default()),
12506            RestartPolicy::new(1, Duration::ZERO),
12507        );
12508        let module = supervisor
12509            .spawn(ModuleSpec {
12510                module_id: "none-clean-exit-budget".to_string(),
12511                program: fake_aft_stub_path(),
12512                args: Vec::new(),
12513                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12514                reserved: false,
12515                reserved_prefixes: Vec::new(),
12516                protocol: ModuleProtocol::None,
12517                overlap: Default::default(),
12518            })
12519            .unwrap();
12520
12521        // Failed follows the terminal write; this deadline only bounds a hang,
12522        // not an assumed duration for the two launches or their exit recording.
12523        let deadline = Instant::now() + Duration::from_secs(10);
12524        loop {
12525            let status = module.status().unwrap();
12526            if status.state == ModuleState::Failed {
12527                break;
12528            }
12529            assert!(
12530                Instant::now() < deadline,
12531                "module never exhausted its budget: {status:?} {:?}",
12532                module.terminal_history()
12533            );
12534            sleep(Duration::from_millis(10)).await;
12535        }
12536        let history = module.terminal_history();
12537        assert_eq!(
12538            history
12539                .entries
12540                .iter()
12541                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12542                .collect::<Vec<_>>(),
12543            vec![
12544                (Some(0), TerminalDisposition::Restarting),
12545                (Some(0), TerminalDisposition::Failed),
12546            ]
12547        );
12548        let detail = history.entries[1]
12549            .disposition_detail
12550            .as_deref()
12551            .expect("a budget failure names the budget");
12552        assert!(detail.contains("max_restarts=1"), "{detail}");
12553        assert_eq!(module.status().unwrap().spawn_generation, 2);
12554    }
12555
12556    #[test]
12557    fn restart_budget_failure_is_published_after_its_terminal_record() {
12558        use std::io::{Read, Write};
12559        let supervisor = Supervisor::new_for_test(
12560            Arc::new(Registry::default()),
12561            RestartPolicy::new(0, Duration::ZERO),
12562        );
12563        let runtime = supervisor.runtime_config();
12564        let spec = ModuleSpec {
12565            protocol: ModuleProtocol::None,
12566            ..windowed_crash_spec("budget-publication-order")
12567        };
12568        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12569            ModuleState::Running,
12570            true,
12571        )));
12572        lock_snapshot(&snapshot).unwrap().process_alive = true;
12573        let reaped_snapshot = snapshot.clone();
12574        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12575        let history = runtime.terminal_ring.clone();
12576        let reap = std::thread::spawn(move || {
12577            tokio::runtime::Builder::new_current_thread()
12578                .enable_all()
12579                .build()
12580                .unwrap()
12581                .block_on(on_child_exit(
12582                    &spec,
12583                    runtime.restart_policy,
12584                    &supervisor.registry,
12585                    &reaped_snapshot,
12586                    &runtime.terminal_ring,
12587                    &runtime.spawn_events,
12588                    &runtime.child_roster,
12589                    ExitReport {
12590                        kind: ExitKind::Clean,
12591                        code: Some(0),
12592                        signal: None,
12593                        at_ms: 1,
12594                    },
12595                    true,
12596                ))
12597        });
12598        let mut reached = [0];
12599        barrier.read_exact(&mut reached).unwrap();
12600        assert_eq!(&reached, b"R");
12601        let (before_state, before_alive) = {
12602            let state = lock_snapshot(&snapshot).unwrap();
12603            (state.state, state.process_alive)
12604        };
12605        let before_history = history.lock().unwrap().snapshot();
12606        barrier.write_all(b"X").unwrap();
12607        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12608        assert!(before_history.entries.is_empty());
12609        assert_ne!(
12610            before_state,
12611            ModuleState::Failed,
12612            "Failed was visible before its terminal record could be written"
12613        );
12614        assert!(before_alive);
12615        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12616        let history = history.lock().unwrap().snapshot();
12617        assert_eq!(history.entries.len(), 1);
12618        assert_eq!(history.entries[0].exit_code, Some(0));
12619        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12620    }
12621
12622    #[test]
12623    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12624        use std::io::{Read, Write};
12625
12626        let supervisor = Supervisor::new_for_test(
12627            Arc::new(Registry::default()),
12628            RestartPolicy::new(3, Duration::ZERO),
12629        );
12630        let mut runtime = supervisor.runtime_config();
12631        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12632        let snapshot = module.inner.snapshot.clone();
12633        lock_snapshot(&snapshot).unwrap().process_alive = true;
12634        runtime.terminal_ring = module.inner.terminal_ring.clone();
12635        let spec = windowed_crash_spec(module.module_id());
12636        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12637        let reaped_snapshot = snapshot.clone();
12638        let reap = std::thread::spawn(move || {
12639            tokio::runtime::Builder::new_current_thread()
12640                .enable_all()
12641                .build()
12642                .unwrap()
12643                .block_on(on_child_exit(
12644                    &spec,
12645                    runtime.restart_policy,
12646                    &supervisor.registry,
12647                    &reaped_snapshot,
12648                    &runtime.terminal_ring,
12649                    &runtime.spawn_events,
12650                    &runtime.child_roster,
12651                    ExitReport {
12652                        kind: ExitKind::Crash,
12653                        code: Some(2),
12654                        signal: None,
12655                        at_ms: 1,
12656                    },
12657                    true,
12658                ))
12659        });
12660
12661        // Hold the crash handler between its budget decision and publication.
12662        // Connection cleanup must still see the recovery that this crash chose,
12663        // even when that recovery consumes the last available restart slot.
12664        let mut reached = [0];
12665        barrier.read_exact(&mut reached).unwrap();
12666        assert_eq!(&reached, b"R");
12667        let before_status = module.status().unwrap();
12668        let before_history = module.terminal_history();
12669        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12670        barrier.write_all(b"X").unwrap();
12671        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12672
12673        assert_eq!(before_status.state, ModuleState::Running);
12674        assert!(before_status.process_alive);
12675        assert_eq!(before_status.restart_count, 3);
12676        assert!(before_history.entries.is_empty());
12677        assert!(
12678            before_recovers,
12679            "connection cleanup would report a terminal close for a scheduled crash restart"
12680        );
12681        let restarting = module.status().unwrap();
12682        assert_eq!(restarting.state, ModuleState::Restarting);
12683        assert_eq!(restarting.restart_count, 3);
12684        assert!(module.will_recover_after_connection_loss().unwrap());
12685        let history = module.terminal_history();
12686        assert_eq!(history.entries.len(), 1);
12687        assert_eq!(
12688            history.entries[0].disposition,
12689            TerminalDisposition::Restarting
12690        );
12691
12692        // Once the replacement is running, its next crash must be judged
12693        // against the exhausted budget rather than the previous recovery.
12694        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12695            state.state = ModuleState::Running;
12696            state.respawn_pending = false;
12697        })
12698        .unwrap();
12699        assert!(!module.will_recover_after_connection_loss().unwrap());
12700        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12701            state.enabled = false;
12702            state.respawn_pending = true;
12703        })
12704        .unwrap();
12705        assert!(!module.will_recover_after_connection_loss().unwrap());
12706    }
12707
12708    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12709        use std::net::{TcpListener, TcpStream};
12710        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12711        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12712        let (barrier, _) = listener.accept().unwrap();
12713        for stream in [&writer, &barrier] {
12714            stream
12715                .set_read_timeout(Some(Duration::from_secs(10)))
12716                .unwrap();
12717            stream
12718                .set_write_timeout(Some(Duration::from_secs(10)))
12719                .unwrap();
12720        }
12721        *events.1.lock().unwrap() = Some(writer);
12722        barrier
12723    }
12724
12725    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12726        use std::io::{Read, Write};
12727        let barrier = events.1.lock().unwrap().take();
12728        if let Some(mut barrier) = barrier {
12729            barrier.write_all(b"R").unwrap();
12730            let mut release = [0];
12731            barrier.read_exact(&mut release).unwrap();
12732            assert_eq!(&release, b"X");
12733        }
12734    }
12735
12736    #[test]
12737    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12738        use std::io::{Read, Write};
12739
12740        let supervisor = Supervisor::new_for_test(
12741            Arc::new(Registry::default()),
12742            RestartPolicy::new(0, Duration::ZERO),
12743        );
12744        let mut runtime = supervisor.runtime_config();
12745        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12746        let snapshot = module.inner.snapshot.clone();
12747        runtime.terminal_ring = module.inner.terminal_ring.clone();
12748        let spec = ModuleSpec {
12749            module_id: module.module_id().into(),
12750            program: fake_aft_stub_path(),
12751            protocol: ModuleProtocol::None,
12752            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12753            ..windowed_crash_spec(module.module_id())
12754        };
12755        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12756        let reap = std::thread::spawn(move || {
12757            tokio::runtime::Builder::new_current_thread()
12758                .enable_all()
12759                .build()
12760                .unwrap()
12761                .block_on(async {
12762                    let mut child =
12763                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12764                    health_restart_child(
12765                        &spec,
12766                        &runtime,
12767                        &supervisor.registry,
12768                        &supervisor.process_liveness,
12769                        &snapshot,
12770                        &mut child,
12771                        SupervisorHealthStatus::Failing,
12772                        Some("health probe did not answer"),
12773                        1,
12774                    )
12775                    .await
12776                    .unwrap();
12777                    assert!(child.is_none());
12778                });
12779        });
12780        // The loopback barrier acknowledges entry into terminal recording by
12781        // health_restart_child's drain, then waits for the test's release byte.
12782        // Status and history are observed while that writer is held, not after
12783        // an elapsed-time guess about when the health action has completed.
12784        let mut reached = [0];
12785        barrier.read_exact(&mut reached).unwrap();
12786        assert_eq!(&reached, b"R");
12787        let before_status = module.status().unwrap();
12788        let before_history = module.terminal_history();
12789        barrier.write_all(b"X").unwrap();
12790        reap.join().unwrap();
12791        assert!(before_history.entries.is_empty());
12792        assert_ne!(
12793            before_status.state,
12794            ModuleState::Failed,
12795            "Failed was visible before its terminal record could be written"
12796        );
12797        assert!(
12798            before_status.process_alive,
12799            "process death was visible before its terminal record could be written"
12800        );
12801        let failed = module.status().unwrap();
12802        assert_eq!(failed.state, ModuleState::Failed);
12803        assert!(!failed.process_alive);
12804        let history = module.terminal_history();
12805        assert_eq!(history.entries.len(), 1);
12806        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12807        assert_eq!(
12808            history.entries[0].disposition_detail.as_deref(),
12809            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12810        );
12811    }
12812
12813    #[test]
12814    fn observed_exits_publish_process_death_after_terminal_history() {
12815        use std::io::{Read, Write};
12816        for (kind, enabled, shutdown, wait_error, expected_state) in [
12817            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12818            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12819            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12820            (
12821                ExitKind::DeliberateSeverance,
12822                true,
12823                false,
12824                false,
12825                ModuleState::Restarting,
12826            ),
12827            #[cfg(unix)]
12828            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12829            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12830        ] {
12831            let supervisor = Supervisor::new_for_test(
12832                Arc::new(Registry::default()),
12833                RestartPolicy::new(1, Duration::ZERO),
12834            );
12835            let runtime = supervisor.runtime_config();
12836            #[cfg(unix)]
12837            if shutdown {
12838                runtime.child_roster.close();
12839            }
12840            #[cfg(not(unix))]
12841            let _ = shutdown;
12842            let spec = windowed_crash_spec("exit-publication-order");
12843            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12844                ModuleState::Running,
12845                enabled,
12846            )));
12847            lock_snapshot(&snapshot).unwrap().process_alive = true;
12848            let reaped_snapshot = snapshot.clone();
12849            let history = runtime.terminal_ring.clone();
12850            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12851            let reap = std::thread::spawn(move || {
12852                if wait_error {
12853                    super::fail_after_wait_error(
12854                        &spec.module_id,
12855                        &reaped_snapshot,
12856                        &runtime.terminal_ring,
12857                        &runtime.spawn_events,
12858                        "wait failed".into(),
12859                        &runtime.failure_context(true),
12860                    );
12861                } else {
12862                    tokio::runtime::Builder::new_current_thread()
12863                        .enable_all()
12864                        .build()
12865                        .unwrap()
12866                        .block_on(on_child_exit(
12867                            &spec,
12868                            runtime.restart_policy,
12869                            &supervisor.registry,
12870                            &reaped_snapshot,
12871                            &runtime.terminal_ring,
12872                            &runtime.spawn_events,
12873                            &runtime.child_roster,
12874                            ExitReport {
12875                                kind,
12876                                code: Some(0),
12877                                signal: None,
12878                                at_ms: 1,
12879                            },
12880                            true,
12881                        ));
12882                }
12883            });
12884            let mut reached = [0];
12885            barrier.read_exact(&mut reached).unwrap();
12886            assert_eq!(&reached, b"R");
12887            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12888            let before_history = history.lock().unwrap().snapshot();
12889            barrier.write_all(b"X").unwrap();
12890            reap.join().unwrap();
12891            assert!(
12892                before_alive,
12893                "{kind:?}: death was published before recording"
12894            );
12895            assert!(before_history.entries.is_empty());
12896            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12897            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12898            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12899        }
12900    }
12901
12902    #[tokio::test]
12903    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12904        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12905        let path = dir.join("terminals.jsonl");
12906        std::fs::create_dir(&path).unwrap();
12907        let supervisor = Supervisor::new_for_test(
12908            Arc::new(Registry::default()),
12909            RestartPolicy::new(0, Duration::ZERO),
12910        )
12911        .with_terminal_journal(path, "health-budget-journal-failure".into());
12912        let runtime = supervisor.runtime_config();
12913        let spec = ModuleSpec {
12914            program: fake_aft_stub_path(),
12915            protocol: ModuleProtocol::None,
12916            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12917            ..windowed_crash_spec("health-budget-journal-failure")
12918        };
12919        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12920        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12921        health_restart_child(
12922            &spec,
12923            &runtime,
12924            &supervisor.registry,
12925            &supervisor.process_liveness,
12926            &snapshot,
12927            &mut child,
12928            SupervisorHealthStatus::Failing,
12929            None,
12930            1,
12931        )
12932        .await
12933        .unwrap();
12934        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12935        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12936        let history = runtime
12937            .terminal_ring
12938            .lock()
12939            .unwrap()
12940            .durable_history(&spec.module_id);
12941        assert!(history.journal_write_failures > 0);
12942        assert_eq!(history.entries.len(), 1);
12943        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12944    }
12945
12946    #[test]
12947    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12948        let supervisor = Supervisor::new_for_test(
12949            Arc::new(Registry::default()),
12950            RestartPolicy::new(0, Duration::ZERO),
12951        );
12952        let runtime = supervisor.runtime_config();
12953        let spec = ModuleSpec {
12954            program: fake_aft_stub_path(),
12955            protocol: ModuleProtocol::None,
12956            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12957            ..windowed_crash_spec("health-budget-recording-panic")
12958        };
12959        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12960        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12961            tokio::runtime::Builder::new_current_thread()
12962                .enable_all()
12963                .build()
12964                .unwrap()
12965                .block_on(async {
12966                    let mut child =
12967                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12968                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12969                    health_restart_child(
12970                        &spec,
12971                        &runtime,
12972                        &supervisor.registry,
12973                        &supervisor.process_liveness,
12974                        &snapshot,
12975                        &mut child,
12976                        SupervisorHealthStatus::Failing,
12977                        None,
12978                        1,
12979                    )
12980                    .await
12981                    .unwrap();
12982                });
12983        }));
12984        let panic = result.expect_err("recording must still unwind");
12985        assert_eq!(
12986            panic.downcast_ref::<String>().map(String::as_str),
12987            Some("spawn event sequence exhausted")
12988        );
12989        let state = lock_snapshot(&snapshot).unwrap();
12990        assert_eq!(state.state, ModuleState::Failed);
12991        assert!(!state.process_alive);
12992        assert!(state.last_exit.is_some());
12993    }
12994
12995    #[tokio::test]
12996    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12997        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12998        let path = dir.join("terminals.jsonl");
12999        std::fs::create_dir(&path).unwrap();
13000        let supervisor = Supervisor::new_for_test(
13001            Arc::new(Registry::default()),
13002            RestartPolicy::new(0, Duration::ZERO),
13003        )
13004        .with_terminal_journal(path, "budget-journal-failure".into());
13005        let runtime = supervisor.runtime_config();
13006        let spec = windowed_crash_spec("budget-journal-failure");
13007        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13008        assert!(matches!(
13009            on_child_exit(
13010                &spec,
13011                runtime.restart_policy,
13012                &supervisor.registry,
13013                &snapshot,
13014                &runtime.terminal_ring,
13015                &runtime.spawn_events,
13016                &runtime.child_roster,
13017                crash_exit_report(1),
13018                true,
13019            )
13020            .await,
13021            NextAction::Stop { .. }
13022        ));
13023        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13024        let history = runtime
13025            .terminal_ring
13026            .lock()
13027            .unwrap()
13028            .durable_history(&spec.module_id);
13029        assert!(history.journal_write_failures > 0);
13030        assert_eq!(history.entries.len(), 1);
13031        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13032    }
13033
13034    #[test]
13035    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13036        let supervisor = Supervisor::new_for_test(
13037            Arc::new(Registry::default()),
13038            RestartPolicy::new(0, Duration::ZERO),
13039        );
13040        let runtime = supervisor.runtime_config();
13041        let spec = windowed_crash_spec("budget-recording-panic");
13042        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13043        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13044        // Exhausting the event sequence makes emit_exited panic before the
13045        // terminal write, exercising publication on the recording unwind.
13046        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13047        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13048            tokio::runtime::Builder::new_current_thread()
13049                .enable_all()
13050                .build()
13051                .unwrap()
13052                .block_on(on_child_exit(
13053                    &spec,
13054                    runtime.restart_policy,
13055                    &supervisor.registry,
13056                    &snapshot,
13057                    &runtime.terminal_ring,
13058                    &runtime.spawn_events,
13059                    &runtime.child_roster,
13060                    crash_exit_report(1),
13061                    true,
13062                ))
13063        }));
13064        let panic = result.err().expect("recording must still unwind");
13065        assert_eq!(
13066            panic.downcast_ref::<String>().map(String::as_str),
13067            Some("spawn event sequence exhausted")
13068        );
13069        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13070    }
13071
13072    /// A stop the supervisor itself requests still stops a protocol-none
13073    /// module, even though the child answers the SIGTERM with exit 0.
13074    #[cfg(unix)]
13075    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13076    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13077        for disable in [false, true] {
13078            let label = if disable {
13079                "none-requested-disable"
13080            } else {
13081                "none-requested-stop"
13082            };
13083            let dir = cortexkit_test_support::ScratchDir::new(label);
13084            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13085            let supervisor = Supervisor::new_for_test(
13086                Arc::new(Registry::default()),
13087                RestartPolicy::new(3, Duration::ZERO),
13088            );
13089            let module = supervisor.spawn(spec).unwrap();
13090            wait_for_file(&ready).await;
13091
13092            if disable {
13093                module.set_enabled(false).await.unwrap();
13094            } else {
13095                module.stop().await.unwrap();
13096            }
13097            assert!(
13098                marker.exists(),
13099                "{label}: the child must have left through its SIGTERM handler with exit 0"
13100            );
13101
13102            // Long enough for a zero-backoff respawn to have happened if the
13103            // exit had been treated as a crash.
13104            sleep(Duration::from_millis(500)).await;
13105            let status = module.status().unwrap();
13106            let expected = if disable {
13107                ModuleState::Disabled
13108            } else {
13109                ModuleState::Stopped
13110            };
13111            assert_eq!(status.state, expected, "{label}");
13112            assert_eq!(
13113                status.spawn_generation, 1,
13114                "{label}: respawned after a requested stop"
13115            );
13116            let history = module.terminal_history();
13117            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13118            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13119            assert_ne!(
13120                history.entries[0].disposition,
13121                TerminalDisposition::Restarting,
13122                "{label}"
13123            );
13124        }
13125    }
13126
13127    /// A subc-wire module that exits 0 on its own is still a stop: the
13128    /// protocol-none rule must not reach it.
13129    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13130    async fn subc_wire_clean_exit_is_still_a_stop() {
13131        let supervisor = Supervisor::new_for_test(
13132            Arc::new(Registry::default()),
13133            RestartPolicy::new(3, Duration::ZERO),
13134        );
13135        let module = supervisor
13136            .spawn(ModuleSpec {
13137                module_id: "wire-clean-exit".to_string(),
13138                program: fake_aft_stub_path(),
13139                args: Vec::new(),
13140                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13141                reserved: false,
13142                reserved_prefixes: Vec::new(),
13143                protocol: ModuleProtocol::Subc,
13144                overlap: Default::default(),
13145            })
13146            .unwrap();
13147
13148        let deadline = Instant::now() + Duration::from_secs(10);
13149        while module.terminal_history().entries.is_empty() {
13150            assert!(Instant::now() < deadline, "module never exited");
13151            sleep(Duration::from_millis(10)).await;
13152        }
13153        // Long enough for a zero-backoff respawn to have happened.
13154        sleep(Duration::from_millis(500)).await;
13155        let status = module.status().unwrap();
13156        assert_eq!(status.state, ModuleState::Stopped);
13157        assert_eq!(status.spawn_generation, 1);
13158        let history = module.terminal_history();
13159        assert_eq!(history.entries.len(), 1, "{history:?}");
13160        assert_eq!(history.entries[0].exit_code, Some(0));
13161        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13162    }
13163
13164    #[cfg(unix)]
13165    #[tokio::test]
13166    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13167        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13168        let record = dir.join("live-children.json");
13169        let supervisor = Supervisor::new_for_test(
13170            Arc::new(Registry::default()),
13171            RestartPolicy::new(0, Duration::ZERO),
13172        );
13173        let mut runtime = supervisor.runtime_config();
13174        runtime.child_roster.record_to(record.clone());
13175        let gate = Arc::new(super::ReloadExitRecordGate::default());
13176        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13177        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13178        let spec = ModuleSpec {
13179            module_id: "reload-exit-roster".into(),
13180            program: fake_aft_stub_path(),
13181            args: Vec::new(),
13182            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13183            reserved: false,
13184            reserved_prefixes: Vec::new(),
13185            protocol: ModuleProtocol::Subc,
13186            overlap: Default::default(),
13187        };
13188        let mut child = None;
13189        let reload = super::finish_reload_child(
13190            &spec,
13191            &runtime,
13192            &supervisor.registry,
13193            &supervisor.process_liveness,
13194            &snapshot,
13195            &mut child,
13196        );
13197        tokio::pin!(reload);
13198        tokio::select! {
13199            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13200            _ = gate.reached.notified() => {}
13201        }
13202        assert!(runtime
13203            .terminal_ring
13204            .lock()
13205            .unwrap()
13206            .snapshot()
13207            .entries
13208            .is_empty());
13209        assert_eq!(
13210            crate::live_children::read_record(&record).unwrap().len(),
13211            1,
13212            "shutdown must still wait for the reaped child until its terminal record exists"
13213        );
13214        runtime.child_roster.close();
13215        gate.resume.notify_one();
13216        assert!(reload.await.is_err());
13217        assert!(crate::live_children::read_record(&record)
13218            .unwrap()
13219            .is_empty());
13220        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13221        assert_eq!(history.entries.len(), 1);
13222        assert_eq!(
13223            history.entries[0].disposition,
13224            TerminalDisposition::DaemonShutdown
13225        );
13226    }
13227
13228    /// Each restart-producing arm has its own state transition. Keeping their
13229    /// lifetime count assertions adjacent prevents a later new arm from silently
13230    /// spending budget without recording the historical restart.
13231    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13232    async fn every_restart_increment_path_advances_lifetime_count() {
13233        let supervisor = Supervisor::new_for_test(
13234            Arc::new(Registry::default()),
13235            RestartPolicy::new(1, Duration::ZERO),
13236        );
13237        let runtime = supervisor.runtime_config();
13238        let spec = ModuleSpec {
13239            module_id: "lifetime-increment-path".to_string(),
13240            program: PathBuf::from("/unused/lifetime-increment-path"),
13241            args: Vec::new(),
13242            env: Vec::new(),
13243            reserved: false,
13244            reserved_prefixes: Vec::new(),
13245            protocol: ModuleProtocol::Subc,
13246            overlap: Default::default(),
13247        };
13248
13249        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13250        assert!(matches!(
13251            on_child_exit(
13252                &spec,
13253                runtime.restart_policy,
13254                &supervisor.registry,
13255                &crash_snapshot,
13256                &runtime.terminal_ring,
13257                &runtime.spawn_events,
13258                &runtime.child_roster,
13259                ExitReport {
13260                    kind: ExitKind::Crash,
13261                    code: Some(1),
13262                    signal: None,
13263                    at_ms: 1,
13264                },
13265                true,
13266            )
13267            .await,
13268            NextAction::Restart { schedule: _ }
13269        ));
13270        let (crash_restarts, crash_lifetime) = {
13271            let state = lock_snapshot(&crash_snapshot).unwrap();
13272            (state.crash_restarts.len(), state.lifetime_restarts)
13273        };
13274        assert_eq!(crash_restarts, 1);
13275        assert_eq!(crash_lifetime, 1);
13276
13277        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13278        let mut health_child = None;
13279        assert!(matches!(
13280            health_restart_child(
13281                &spec,
13282                &runtime,
13283                &supervisor.registry,
13284                &supervisor.process_liveness,
13285                &health_snapshot,
13286                &mut health_child,
13287                SupervisorHealthStatus::Failing,
13288                None,
13289                2,
13290            )
13291            .await,
13292            Ok(())
13293        ));
13294        assert!(health_child.is_none());
13295        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13296        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13297        let (health_restarts, health_lifetime) = {
13298            let state = lock_snapshot(&health_snapshot).unwrap();
13299            (state.crash_restarts.len(), state.lifetime_restarts)
13300        };
13301        assert_eq!(health_restarts, 1);
13302        assert_eq!(health_lifetime, 1);
13303
13304        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13305        let mut reload_child = None;
13306        assert!(matches!(
13307            handle_reload_spawn_failure(
13308                &spec,
13309                &runtime,
13310                &supervisor.process_liveness,
13311                &reload_snapshot,
13312                &mut reload_child,
13313                "forced reload spawn failure".to_string(),
13314            )
13315            .await,
13316            Err(SuperviseError::ReloadFailed { .. })
13317        ));
13318        let (reload_restarts, reload_lifetime) = {
13319            let state = lock_snapshot(&reload_snapshot).unwrap();
13320            (state.crash_restarts.len(), state.lifetime_restarts)
13321        };
13322        assert_eq!(reload_restarts, 1);
13323        assert_eq!(reload_lifetime, 1);
13324    }
13325
13326    #[tokio::test]
13327    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13328        let supervisor = Supervisor::new_for_test(
13329            Arc::new(Registry::default()),
13330            RestartPolicy::new(3, Duration::ZERO),
13331        );
13332        let runtime = supervisor.runtime_config();
13333        let spec = ModuleSpec {
13334            module_id: "deliberately-severed".to_string(),
13335            program: PathBuf::from("/unused/deliberately-severed"),
13336            args: Vec::new(),
13337            env: Vec::new(),
13338            reserved: false,
13339            reserved_prefixes: Vec::new(),
13340            protocol: ModuleProtocol::Subc,
13341            overlap: Default::default(),
13342        };
13343        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13344        let process = ProcessIdentity {
13345            pid: 41,
13346            start_time: 101,
13347        };
13348        record_deliberate_severance(&snapshot, process).unwrap();
13349        let exit_report = apply_deliberate_severance_marker(
13350            &snapshot,
13351            Some(process),
13352            ExitReport {
13353                kind: ExitKind::Crash,
13354                code: Some(1),
13355                signal: None,
13356                at_ms: 1,
13357            },
13358        );
13359        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13360
13361        assert!(matches!(
13362            on_child_exit(
13363                &spec,
13364                runtime.restart_policy,
13365                &supervisor.registry,
13366                &snapshot,
13367                &runtime.terminal_ring,
13368                &runtime.spawn_events,
13369                &runtime.child_roster,
13370                exit_report,
13371                true,
13372            )
13373            .await,
13374            NextAction::Restart { schedule: _ }
13375        ));
13376        let state = lock_snapshot(&snapshot).unwrap();
13377        assert_eq!(state.lifetime_restarts, 1);
13378        assert_eq!(state.crash_restarts.len(), 0);
13379    }
13380
13381    #[tokio::test]
13382    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13383        let supervisor = Supervisor::new_for_test(
13384            Arc::new(Registry::default()),
13385            RestartPolicy::new(3, Duration::ZERO),
13386        );
13387        let runtime = supervisor.runtime_config();
13388        let spec = ModuleSpec {
13389            module_id: "genuine-crash".to_string(),
13390            program: PathBuf::from("/unused/genuine-crash"),
13391            args: Vec::new(),
13392            env: Vec::new(),
13393            reserved: false,
13394            reserved_prefixes: Vec::new(),
13395            protocol: ModuleProtocol::Subc,
13396            overlap: Default::default(),
13397        };
13398        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13399
13400        assert!(matches!(
13401            on_child_exit(
13402                &spec,
13403                runtime.restart_policy,
13404                &supervisor.registry,
13405                &snapshot,
13406                &runtime.terminal_ring,
13407                &runtime.spawn_events,
13408                &runtime.child_roster,
13409                ExitReport {
13410                    kind: ExitKind::Crash,
13411                    code: Some(1),
13412                    signal: None,
13413                    at_ms: 1,
13414                },
13415                true,
13416            )
13417            .await,
13418            NextAction::Restart { schedule: _ }
13419        ));
13420        let state = lock_snapshot(&snapshot).unwrap();
13421        assert_eq!(state.lifetime_restarts, 1);
13422        assert_eq!(state.crash_restarts.len(), 1);
13423    }
13424
13425    fn crash_exit_report(at_ms: u64) -> ExitReport {
13426        ExitReport {
13427            kind: ExitKind::Crash,
13428            code: Some(1),
13429            signal: None,
13430            at_ms,
13431        }
13432    }
13433
13434    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13435        ModuleSpec {
13436            module_id: module_id.to_string(),
13437            program: PathBuf::from("/unused").join(module_id),
13438            args: Vec::new(),
13439            env: Vec::new(),
13440            reserved: false,
13441            reserved_prefixes: Vec::new(),
13442            protocol: ModuleProtocol::Subc,
13443            overlap: Default::default(),
13444        }
13445    }
13446
13447    /// A real crash loop still stops. Three crashes with nothing aging out spend
13448    /// a budget of two and the third respawn is refused, and both surfaces an
13449    /// operator has -- the log line and the retained terminal record -- name the
13450    /// window rather than only the cap, because `max_restarts=2` alone is what
13451    /// this budget used to mean.
13452    #[tokio::test]
13453    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13454        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13455        let supervisor = Supervisor::new_for_test(
13456            Arc::new(Registry::default()),
13457            RestartPolicy::new(2, Duration::ZERO),
13458        );
13459        let runtime = supervisor.runtime_config();
13460        let spec = windowed_crash_spec("crash-loop-in-window");
13461        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13462
13463        for attempt in 1..=2 {
13464            assert!(
13465                matches!(
13466                    on_child_exit(
13467                        &spec,
13468                        runtime.restart_policy,
13469                        &supervisor.registry,
13470                        &snapshot,
13471                        &runtime.terminal_ring,
13472                        &runtime.spawn_events,
13473                        &runtime.child_roster,
13474                        crash_exit_report(attempt),
13475                        true,
13476                    )
13477                    .await,
13478                    NextAction::Restart { schedule: _ }
13479                ),
13480                "crash {attempt} is inside the budget and must respawn"
13481            );
13482        }
13483
13484        assert!(matches!(
13485            on_child_exit(
13486                &spec,
13487                runtime.restart_policy,
13488                &supervisor.registry,
13489                &snapshot,
13490                &runtime.terminal_ring,
13491                &runtime.spawn_events,
13492                &runtime.child_roster,
13493                crash_exit_report(3),
13494                true,
13495            )
13496            .await,
13497            NextAction::Stop { .. }
13498        ));
13499
13500        {
13501            let state = lock_snapshot(&snapshot).unwrap();
13502            assert_eq!(state.state, ModuleState::Failed);
13503            assert_eq!(state.crash_restarts.len(), 2);
13504            assert_eq!(state.lifetime_restarts, 2);
13505        }
13506
13507        let history = runtime
13508            .terminal_ring
13509            .lock()
13510            .expect("terminal ring is not poisoned")
13511            .snapshot();
13512        let last = history
13513            .entries
13514            .last()
13515            .expect("the refused crash is retained");
13516        assert_eq!(last.disposition, TerminalDisposition::Failed);
13517        assert_eq!(
13518            last.disposition_detail.as_deref(),
13519            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13520        );
13521
13522        let captured = crate::router::test_log::captured_logs(&logs);
13523        assert!(
13524            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13525            "the stop must be logged with its window: {captured}"
13526        );
13527    }
13528
13529    /// The rate, stated as a test: three crashes where the first has aged past
13530    /// the window are two crashes as far as the budget is concerned, so the
13531    /// third respawn is allowed and the ring holds only the two recent ones.
13532    ///
13533    /// This is the case a lifetime counter got wrong -- and the case the daemon
13534    /// now hits routinely, since a module exits non-zero every time its
13535    /// connection to the daemon drops.
13536    #[tokio::test]
13537    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13538        let supervisor = Supervisor::new_for_test(
13539            Arc::new(Registry::default()),
13540            RestartPolicy::new(2, Duration::ZERO),
13541        );
13542        let runtime = supervisor.runtime_config();
13543        let spec = windowed_crash_spec("crash-across-windows");
13544        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13545
13546        for attempt in 1..=2 {
13547            assert!(matches!(
13548                on_child_exit(
13549                    &spec,
13550                    runtime.restart_policy,
13551                    &supervisor.registry,
13552                    &snapshot,
13553                    &runtime.terminal_ring,
13554                    &runtime.spawn_events,
13555                    &runtime.child_roster,
13556                    crash_exit_report(attempt),
13557                    true,
13558                )
13559                .await,
13560                NextAction::Restart { schedule: _ }
13561            ));
13562        }
13563
13564        // The oldest crash moves out of the window; nothing else about the
13565        // module changes.
13566        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13567            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13568        })
13569        .unwrap();
13570
13571        assert!(
13572            matches!(
13573                on_child_exit(
13574                    &spec,
13575                    runtime.restart_policy,
13576                    &supervisor.registry,
13577                    &snapshot,
13578                    &runtime.terminal_ring,
13579                    &runtime.spawn_events,
13580                    &runtime.child_roster,
13581                    crash_exit_report(3),
13582                    true,
13583                )
13584                .await,
13585                NextAction::Restart { schedule: _ }
13586            ),
13587            "a crash older than the window must not hold a budget slot"
13588        );
13589
13590        let state = lock_snapshot(&snapshot).unwrap();
13591        assert_eq!(state.state, ModuleState::Restarting);
13592        assert_eq!(
13593            state.crash_restarts.len(),
13594            2,
13595            "the aged instant is dropped and the new one takes its place"
13596        );
13597        assert_eq!(
13598            state.lifetime_restarts, 3,
13599            "the ledger counts every restart, including the ones the window forgot"
13600        );
13601    }
13602
13603    /// An operator restart hands the budget back whole, and the ledger keeps
13604    /// counting. Those are different questions -- "how close is this module to
13605    /// being stopped" and "how many times has it been replaced" -- and the
13606    /// operator action answers only the first.
13607    #[tokio::test]
13608    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13609        let supervisor = Supervisor::new_for_test(
13610            Arc::new(Registry::default()),
13611            RestartPolicy::new(2, Duration::ZERO),
13612        );
13613        let runtime = supervisor.runtime_config();
13614        let spec = windowed_crash_spec("operator-cleared-budget");
13615        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13616
13617        for attempt in 1..=2 {
13618            assert!(matches!(
13619                on_child_exit(
13620                    &spec,
13621                    runtime.restart_policy,
13622                    &supervisor.registry,
13623                    &snapshot,
13624                    &runtime.terminal_ring,
13625                    &runtime.spawn_events,
13626                    &runtime.child_roster,
13627                    crash_exit_report(attempt),
13628                    true,
13629                )
13630                .await,
13631                NextAction::Restart { schedule: _ }
13632            ));
13633        }
13634
13635        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13636        {
13637            let state = lock_snapshot(&snapshot).unwrap();
13638            assert!(
13639                state.crash_restarts.is_empty(),
13640                "an operator restart returns the full budget"
13641            );
13642            assert_eq!(
13643                state.lifetime_restarts, 2,
13644                "clearing the budget must not unmake the crashes"
13645            );
13646        }
13647
13648        assert!(
13649            matches!(
13650                on_child_exit(
13651                    &spec,
13652                    runtime.restart_policy,
13653                    &supervisor.registry,
13654                    &snapshot,
13655                    &runtime.terminal_ring,
13656                    &runtime.spawn_events,
13657                    &runtime.child_roster,
13658                    crash_exit_report(3),
13659                    true,
13660                )
13661                .await,
13662                NextAction::Restart { schedule: _ }
13663            ),
13664            "the cleared budget must be spendable again"
13665        );
13666        let state = lock_snapshot(&snapshot).unwrap();
13667        assert_eq!(state.crash_restarts.len(), 1);
13668        assert_eq!(state.lifetime_restarts, 3);
13669    }
13670
13671    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13672    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13673        let severed = ProcessIdentity {
13674            pid: 41,
13675            start_time: 101,
13676        };
13677        let successor = ProcessIdentity {
13678            pid: 41,
13679            start_time: 202,
13680        };
13681        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13682        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13683            state.pid = Some(successor.pid);
13684            state.process_start_time = Some(successor.start_time);
13685        })
13686        .unwrap();
13687        assert!(!module.record_deliberate_severance(severed).unwrap());
13688
13689        let exit_report = apply_deliberate_severance_marker(
13690            &module.inner.snapshot,
13691            Some(successor),
13692            ExitReport {
13693                kind: ExitKind::Crash,
13694                code: Some(1),
13695                signal: None,
13696                at_ms: 1,
13697            },
13698        );
13699
13700        assert_eq!(exit_report.kind, ExitKind::Crash);
13701    }
13702
13703    #[tokio::test]
13704    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13705        let registry = Registry::default();
13706        let supervisor = Supervisor::new_for_test(
13707            Arc::new(Registry::default()),
13708            RestartPolicy::new(3, Duration::ZERO),
13709        );
13710        let runtime = supervisor.runtime_config();
13711        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13712        let spec = ModuleSpec {
13713            module_id: "drain-deliberate-severance".to_string(),
13714            program: fake_aft_stub_path(),
13715            args: Vec::new(),
13716            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13717            reserved: false,
13718            reserved_prefixes: Vec::new(),
13719            protocol: ModuleProtocol::Subc,
13720            overlap: Default::default(),
13721        };
13722        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13723        let process = ProcessIdentity {
13724            pid: 41,
13725            start_time: 101,
13726        };
13727        child.process_identity = Some(process);
13728        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13729            state.pid = Some(process.pid);
13730            state.process_start_time = Some(process.start_time);
13731        })
13732        .unwrap();
13733        record_deliberate_severance(&snapshot, process).unwrap();
13734
13735        drain_child_to_state(
13736            &spec.module_id,
13737            spec.protocol,
13738            // The child exits on its own; no signal may change the exit this
13739            // test classifies.
13740            StopNotice::SentOverConnection,
13741            &registry,
13742            None,
13743            &snapshot,
13744            &runtime.terminal_ring,
13745            &runtime.spawn_events,
13746            child,
13747            Duration::from_secs(1),
13748            ModuleState::Stopped,
13749            Some(false),
13750            &super::FailureContext::operator(),
13751        )
13752        .await
13753        .unwrap();
13754
13755        let state = lock_snapshot(&snapshot).unwrap();
13756        assert_eq!(
13757            state.last_exit.as_ref().map(|exit| exit.kind),
13758            Some(ExitKind::DeliberateSeverance)
13759        );
13760        assert_eq!(state.lifetime_restarts, 1);
13761        assert_eq!(state.crash_restarts.len(), 0);
13762        drop(state);
13763        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13764        assert_eq!(
13765            history.entries[0].exit_kind,
13766            subc_control::TerminalExitKind::DeliberateSeverance
13767        );
13768    }
13769
13770    #[tokio::test]
13771    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13772        let registry = Registry::default();
13773        let supervisor = Supervisor::new_for_test(
13774            Arc::new(Registry::default()),
13775            RestartPolicy::new(3, Duration::ZERO),
13776        );
13777        let runtime = supervisor.runtime_config();
13778        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13779        let spec = ModuleSpec {
13780            module_id: "ordinary-drain".to_string(),
13781            program: fake_aft_stub_path(),
13782            args: Vec::new(),
13783            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13784            reserved: false,
13785            reserved_prefixes: Vec::new(),
13786            protocol: ModuleProtocol::Subc,
13787            overlap: Default::default(),
13788        };
13789        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13790
13791        drain_child_to_state(
13792            &spec.module_id,
13793            spec.protocol,
13794            // The child exits on its own; no signal may change the exit this
13795            // test classifies.
13796            StopNotice::SentOverConnection,
13797            &registry,
13798            None,
13799            &snapshot,
13800            &runtime.terminal_ring,
13801            &runtime.spawn_events,
13802            child,
13803            Duration::from_secs(1),
13804            ModuleState::Stopped,
13805            Some(false),
13806            &super::FailureContext::operator(),
13807        )
13808        .await
13809        .unwrap();
13810
13811        let state = lock_snapshot(&snapshot).unwrap();
13812        assert_eq!(
13813            state.last_exit.as_ref().map(|exit| exit.kind),
13814            Some(ExitKind::Crash)
13815        );
13816        assert_eq!(state.lifetime_restarts, 0);
13817        assert_eq!(state.crash_restarts.len(), 0);
13818    }
13819
13820    #[test]
13821    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13822        // The server's generic fatal-routing branch only knows that the
13823        // connection failed; it does not know that the daemon deliberately
13824        // initiated a process-killing severance. Keep this seam explicit so a
13825        // future connection error path cannot silently reintroduce the stale
13826        // exemption that mislabels a later genuine crash.
13827        assert!(!include_str!("server.rs")
13828            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13829    }
13830
13831    /// The `route.closed` `drained` value must be the quiescence wait's own
13832    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13833    /// measurement at all and `false` is the one honest constant. This is the exact
13834    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13835    /// on every return path, including the one that used to return early via `?`
13836    /// with `route.closing` already sent and no `route.closed` ever following.
13837    #[test]
13838    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13839        assert!(drained_after_quiescence_wait(&Ok(true)));
13840        assert!(!drained_after_quiescence_wait(&Ok(false)));
13841        assert!(!drained_after_quiescence_wait(&Err(
13842            SuperviseError::StatePoisoned { module_id: None }
13843        )));
13844    }
13845
13846    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13847    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13848    /// already reaped out-of-band) still leaves a terminal record rather than none
13849    /// at all. Triggering the real `wait()` I/O error from an integration test would
13850    /// need a genuine already-reaped-child race, which is OS-specific and not
13851    /// something this suite attempts elsewhere; this test instead verifies the
13852    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13853    /// the call site itself is verified by inspection to sit in that exact arm.
13854    #[test]
13855    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13856        let ring = Arc::new(Mutex::new(TerminalRing::new(
13857            TerminalRingConfig::default(),
13858            0,
13859        )));
13860        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13861
13862        let snapshot = ring.lock().unwrap().snapshot();
13863        assert_eq!(snapshot.entries.len(), 1);
13864        let entry = &snapshot.entries[0];
13865        assert_eq!(entry.exit_code, None);
13866        assert_eq!(entry.exit_signal, None);
13867        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13868    }
13869
13870    #[test]
13871    fn wait_error_exit_path_preserves_spawn_event_density() {
13872        let feed = super::SpawnEventFeed::default();
13873        feed.configure_incarnation("wait-error-density".to_string());
13874        feed.emit_spawned("wait-error", 41, 1);
13875        let ring = Arc::new(Mutex::new(TerminalRing::new(
13876            TerminalRingConfig::default(),
13877            0,
13878        )));
13879
13880        record_wait_error_terminal("wait-error", &ring, &feed);
13881        feed.emit_spawned("after-wait-error", 42, 2);
13882
13883        let state = feed.0.lock().unwrap();
13884        let sequences = state
13885            .events
13886            .iter()
13887            .map(|event| event.cursor.seq)
13888            .collect::<Vec<_>>();
13889        assert_eq!(sequences, vec![1, 2, 3]);
13890        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13891        assert_eq!(state.events[1].exit_code, None);
13892        assert_eq!(state.events[1].exit_signal, None);
13893    }
13894
13895    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13896    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13897    /// not a clean exit it never actually observed.
13898    #[test]
13899    fn wait_error_exit_report_is_classified_as_a_crash() {
13900        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13901    }
13902}
13903
13904#[cfg(all(test, unix))]
13905mod restart_ordering_tests {
13906    use super::*;
13907    use std::io::Write;
13908    use tokio::io::AsyncReadExt;
13909
13910    #[test]
13911    #[ignore = "child process fixture"]
13912    fn serving_child() {
13913        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13914            return;
13915        };
13916        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13917        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13918        for stream in listener.incoming() {
13919            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13920        }
13921    }
13922
13923    async fn restart_before_publication(requests: usize) {
13924        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13925        let address_file = home.path().join("address");
13926        let spec = ModuleSpec {
13927            module_id: "restart-serving-child".into(),
13928            program: std::env::current_exe().unwrap(),
13929            args: vec![
13930                "--exact".into(),
13931                "supervise::restart_ordering_tests::serving_child".into(),
13932                "--ignored".into(),
13933                "--nocapture".into(),
13934            ],
13935            env: vec![(
13936                "RESTART_TEST_ADDRESS_FILE".into(),
13937                address_file.display().to_string(),
13938            )],
13939            reserved: false,
13940            reserved_prefixes: vec![],
13941            protocol: ModuleProtocol::None,
13942            overlap: Default::default(),
13943        };
13944        let mut supervisor =
13945            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13946        let (spawned_tx, spawned_rx) = oneshot::channel();
13947        let spawned_tx = Mutex::new(Some(spawned_tx));
13948        let (release_tx, release_rx) = std::sync::mpsc::channel();
13949        let release_rx = Mutex::new(release_rx);
13950        // Hold publication, not the child: it must answer a real request before
13951        // the restart enters the queue, even though status still has generation 0.
13952        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13953            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13954            release_rx
13955                .lock()
13956                .unwrap()
13957                .recv_timeout(Duration::from_secs(20))
13958                .unwrap();
13959        })));
13960        let registered = supervisor
13961            .register_configured_with_health(
13962                spec.clone(),
13963                true,
13964                HealthConfig::default(),
13965                None,
13966                RestartPolicy::new(3, Duration::ZERO),
13967                DEFAULT_REGISTER_TIMEOUT,
13968            )
13969            .unwrap();
13970        let module = registered.module.clone();
13971        let startup = tokio::task::spawn_blocking(move || {
13972            supervisor.start_registered_configured(registered, None, true)
13973        });
13974        spawned_rx.await.unwrap();
13975        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13976            let address = loop {
13977                if let Ok(address) = std::fs::read_to_string(&address_file) {
13978                    if !address.is_empty() {
13979                        break address;
13980                    }
13981                }
13982                tokio::time::sleep(Duration::from_millis(1)).await;
13983            };
13984            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13985            let mut reply = String::new();
13986            stream.read_to_string(&mut reply).await.unwrap();
13987            reply.trim().parse::<u32>().unwrap()
13988        })
13989        .await
13990        .expect("child must serve before generation publication");
13991        assert_eq!(
13992            lock_snapshot(&module.inner.snapshot)
13993                .unwrap()
13994                .spawn_generation,
13995            0
13996        );
13997        let mut replies = Vec::new();
13998        for _ in 0..requests {
13999            let mut restart = Box::pin(module.restart(Some(0)));
14000            // Poll the public API through queue admission, leaving only its ACK
14001            // pending. The monitor cannot consume the command until released.
14002            std::future::poll_fn(|cx| {
14003                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14004                std::task::Poll::Ready(())
14005            })
14006            .await;
14007            replies.push(restart);
14008        }
14009        release_tx.send(()).unwrap();
14010        startup.await.unwrap().unwrap();
14011        for reply in replies {
14012            reply.await.unwrap();
14013        }
14014        // Restart schedules its replacement after completing the handler. Wait
14015        // for that separate actor turn before placing the command barrier.
14016        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14017            loop {
14018                if lock_snapshot(&module.inner.snapshot)
14019                    .unwrap()
14020                    .spawn_generation
14021                    > 1
14022                {
14023                    break;
14024                }
14025                tokio::time::sleep(Duration::from_millis(1)).await;
14026            }
14027        })
14028        .await;
14029        // Commands are handled serially. The update-spec reply proves every
14030        // restart handler has finished before we inspect the snapshot; restart's
14031        // own ACK only acknowledges initiation, not replacement completion.
14032        module.update_spec_for_test(spec).await.unwrap();
14033        let (generation, pid) = {
14034            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14035            (state.spawn_generation, state.pid)
14036        };
14037        module.stop().await.unwrap();
14038        replacement.expect("restart must publish a replacement for the already-serving child");
14039        assert_ne!(
14040            pid,
14041            Some(old_pid),
14042            "a serving child predating the restart must be replaced"
14043        );
14044        assert_eq!(generation, 2, "queued restarts must share one replacement");
14045    }
14046
14047    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14048    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14049        restart_before_publication(1).await;
14050    }
14051
14052    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14053    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14054        restart_before_publication(2).await;
14055    }
14056
14057    #[tokio::test]
14058    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14059        let supervisor =
14060            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14061        let registered = supervisor
14062            .register_configured_with_health(
14063                ModuleSpec {
14064                    module_id: "queued-before-replacement".into(),
14065                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14066                    // macOS a module whose running image is not the configured
14067                    // program is refused once its exec is confirmed (see
14068                    // `record_write_stall_tests::sleeping_spec`).
14069                    program: "/bin/sleep".into(),
14070                    args: vec!["600".into()],
14071                    env: vec![],
14072                    reserved: false,
14073                    reserved_prefixes: vec![],
14074                    protocol: ModuleProtocol::None,
14075                    overlap: Default::default(),
14076                },
14077                true,
14078                HealthConfig::default(),
14079                None,
14080                RestartPolicy::default(),
14081                DEFAULT_REGISTER_TIMEOUT,
14082            )
14083            .unwrap();
14084        let RegisteredConfiguredModule {
14085            mut spec,
14086            mut runtime,
14087            module,
14088            mut commands,
14089        } = registered;
14090        let snapshot = &module.inner.snapshot;
14091        let mut restart = Box::pin(module.restart(Some(0)));
14092        std::future::poll_fn(|cx| {
14093            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14094            std::task::Poll::Ready(())
14095        })
14096        .await;
14097        // Drive the actor explicitly so a replacement is definitely spawned
14098        // after receipt, but before this queued command is dequeued.
14099        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14100        let replacement_pid = child.as_ref().unwrap().id();
14101        let command = commands.recv().await.unwrap();
14102        assert!(
14103            handle_supervisor_command(
14104                command,
14105                &mut spec,
14106                &mut runtime,
14107                &supervisor.registry,
14108                &supervisor.process_liveness,
14109                snapshot,
14110                &mut child,
14111                &mut commands,
14112                &mut VecDeque::new(),
14113            )
14114            .await
14115        );
14116        restart.await.unwrap();
14117        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14118        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14119        if let Some(mut child) = child {
14120            child.start_kill().unwrap();
14121            child.wait().await.unwrap();
14122        }
14123        assert_eq!(
14124            kept_pid, replacement_pid,
14125            "a newer replacement must survive"
14126        );
14127        assert!(
14128            !pending,
14129            "a satisfied restart must not schedule another spawn"
14130        );
14131    }
14132}
14133
14134#[cfg(all(test, unix))]
14135mod health_event_tests {
14136    use super::*;
14137    use cortexkit_test_support::ScratchDir;
14138    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14139
14140    struct Actor {
14141        module: SupervisedModule,
14142        registry: Arc<Registry>,
14143        forwarding: Arc<ForwardingTable>,
14144        spec: ModuleSpec,
14145        health: HealthConfig,
14146        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14147        roster: ChildRoster,
14148        _home: ScratchDir,
14149    }
14150
14151    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14152        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14153        // for an observed condition, rather than letting paused time auto-advance
14154        // deadlines or assuming a fixed number of yields finishes the work.
14155        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14156        let tick = Instant::now();
14157        loop {
14158            if let Some(value) = observe() {
14159                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14160                return value;
14161            }
14162            assert!(
14163                std::time::Instant::now() < deadline,
14164                "timed out waiting for {reason}"
14165            );
14166            tokio::task::yield_now().await;
14167        }
14168    }
14169
14170    impl Actor {
14171        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14172            Self::start_with_policy(
14173                protocol,
14174                cadence,
14175                DEFAULT_REGISTER_TIMEOUT,
14176                RestartPolicy::new(3, Duration::ZERO),
14177            )
14178            .await
14179        }
14180
14181        async fn start_with_policy(
14182            protocol: ModuleProtocol,
14183            cadence: Duration,
14184            register_timeout: Duration,
14185            policy: RestartPolicy,
14186        ) -> Self {
14187            let home = ScratchDir::new("health-events");
14188            let registry = Arc::new(Registry::default());
14189            let forwarding = Arc::new(ForwardingTable::default());
14190            let health = HealthConfig {
14191                cadence,
14192                deadline: Duration::from_secs(3600),
14193                ..HealthConfig::default()
14194            };
14195            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14196                .with_forwarding(forwarding.clone())
14197                .with_health_config(health.clone());
14198            let spec = ModuleSpec {
14199                module_id: "event-health-child".into(),
14200                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14201                // the privacy trampoline refuses a module whose running image
14202                // is not the configured program, and `start` below waits for
14203                // the process to be confirmed (`reported_pid`). The shell form
14204                // passed only while the image check ran before the shell had
14205                // replaced itself with `sleep` (see
14206                // `record_write_stall_tests::sleeping_spec`).
14207                program: PathBuf::from("/bin/sleep"),
14208                // The process stays alive but has no real bus peer. Registrations
14209                // below exercise the same registry writes as accepted HELLOs.
14210                args: vec!["600".into()],
14211                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14212                    .into_iter()
14213                    .map(|key| (key.into(), home.path().display().to_string()))
14214                    .collect(),
14215                reserved: false,
14216                reserved_prefixes: vec![],
14217                protocol,
14218                overlap: Default::default(),
14219            };
14220            let module = supervisor
14221                .supervise_configured_requested(
14222                    spec.clone(),
14223                    true,
14224                    health.clone(),
14225                    Some(0),
14226                    policy,
14227                    register_timeout,
14228                    None,
14229                )
14230                .unwrap();
14231            let (_, rx) = mpsc::channel(8);
14232            let actor = Self {
14233                module,
14234                registry,
14235                forwarding,
14236                spec,
14237                health,
14238                rx,
14239                roster: supervisor.child_roster.clone(),
14240                _home: home,
14241            };
14242            actor
14243                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14244                .await;
14245            actor
14246        }
14247
14248        async fn expire_registration_wait(&self, remaining: Duration) {
14249            tokio::time::advance(remaining).await;
14250            wait_for("registration timeout TERM grace", || {
14251                lock_snapshot(&self.module.inner.snapshot)
14252                    .unwrap()
14253                    .actor_select
14254                    .is_none()
14255                    .then_some(())
14256            })
14257            .await;
14258            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14259        }
14260
14261        fn turns(&self) -> u64 {
14262            lock_snapshot(&self.module.inner.snapshot)
14263                .unwrap()
14264                .actor_turns
14265        }
14266
14267        async fn wait_for_select(
14268            &self,
14269            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14270        ) -> ActorSelectCheckpoint {
14271            wait_for("actor select checkpoint", || {
14272                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14273                state.actor_select.filter(|checkpoint| {
14274                    checkpoint.generation == state.spawn_generation
14275                        && state.state == ModuleState::Running
14276                        && state.process_alive
14277                        && state.reported_pid().is_some()
14278                        && expected(checkpoint)
14279                })
14280            })
14281            .await
14282        }
14283
14284        fn hello(&mut self, connection: u64, health: bool) {
14285            let manifest =
14286                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14287                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14288                    .build();
14289            let connection = ConnectionId::new(connection);
14290            let (tx, rx) = mpsc::channel(8);
14291            self.rx = rx;
14292            self.forwarding
14293                .register_module_connection(
14294                    connection,
14295                    self.spec.module_id.clone(),
14296                    subc_protocol::PROTOCOL_VERSION,
14297                    Concurrency::ModuleManaged,
14298                    FrameSink::new(tx),
14299                )
14300                .unwrap();
14301            self.registry
14302                .register_with_control_ops(
14303                    manifest,
14304                    subc_protocol::PROTOCOL_VERSION,
14305                    connection,
14306                    if health {
14307                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14308                    } else {
14309                        vec![]
14310                    },
14311                )
14312                .unwrap();
14313        }
14314
14315        async fn probe(&mut self) -> crate::router::OutboundFrame {
14316            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14317                Ok(frame) => Some(frame),
14318                Err(mpsc::error::TryRecvError::Empty) => None,
14319                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14320            })
14321            .await;
14322            let body: Value = serde_json::from_slice(&frame.body).unwrap();
14323            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14324            frame
14325        }
14326
14327        fn no_probe(&mut self) {
14328            assert!(matches!(
14329                self.rx.try_recv(),
14330                Err(mpsc::error::TryRecvError::Empty)
14331            ));
14332        }
14333
14334        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14335            self.forwarding
14336                .complete_module_control_rpc(
14337                    ConnectionId::new(connection),
14338                    frame.header.corr,
14339                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14340                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14341                        status: HealthStatus::Ok,
14342                        detail: None,
14343                        metrics: None,
14344                    }),
14345                )
14346                .unwrap();
14347        }
14348    }
14349
14350    #[tokio::test(start_paused = true)]
14351    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14352        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14353        let initial = actor.module.status().unwrap();
14354        tokio::time::advance(Duration::from_secs(119)).await;
14355        tokio::task::yield_now().await;
14356        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14357        assert!(actor.module.terminal_history().entries.is_empty());
14358        actor.expire_registration_wait(Duration::from_secs(1)).await;
14359        actor
14360            .wait_for_select(|at| at.generation > initial.spawn_generation)
14361            .await;
14362        let status = actor.module.status().unwrap();
14363        assert_ne!(status.pid, initial.pid);
14364        assert!(!cortexkit_test_support::process_alive(
14365            initial.pid.unwrap() as i32
14366        ));
14367        assert_eq!(status.restart_count, 1);
14368        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14369        let entries = actor.module.terminal_history().entries;
14370        assert_eq!(entries.len(), 1);
14371        assert_eq!(
14372            entries[0].disposition_detail.as_deref(),
14373            Some("registration_timeout")
14374        );
14375        actor.module.stop().await.unwrap();
14376    }
14377
14378    #[tokio::test(start_paused = true)]
14379    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14380        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14381        let pid = actor.module.status().unwrap().pid;
14382        tokio::time::advance(Duration::from_secs(119)).await;
14383        actor.hello(1, false);
14384        let registered = actor
14385            .wait_for_select(|at| at.registered_connection.is_some())
14386            .await;
14387        assert_eq!(
14388            registered.wake_after, None,
14389            "HELLO must cancel the deadline"
14390        );
14391        tokio::time::advance(Duration::from_secs(300)).await;
14392        tokio::task::yield_now().await;
14393        assert_eq!(actor.module.status().unwrap().pid, pid);
14394        assert!(actor.module.terminal_history().entries.is_empty());
14395        // A disconnect after the first HELLO must not re-arm a startup timer.
14396        actor
14397            .registry
14398            .deregister_connection(ConnectionId::new(1))
14399            .unwrap();
14400        actor
14401            .wait_for_select(|at| at.registered_connection.is_none())
14402            .await;
14403        tokio::time::advance(Duration::from_secs(300)).await;
14404        tokio::task::yield_now().await;
14405        assert_eq!(actor.module.status().unwrap().pid, pid);
14406        assert!(actor.module.terminal_history().entries.is_empty());
14407        actor.module.stop().await.unwrap();
14408
14409        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14410        let pid = transient.module.status().unwrap().pid;
14411        tokio::time::advance(Duration::from_secs(119)).await;
14412        let turns = transient.turns();
14413        // Both writes happen before the supervisor can refresh the registry.
14414        // An accepted HELLO must remain evidence after its connection is gone.
14415        transient.hello(1, false);
14416        transient
14417            .registry
14418            .deregister_connection(ConnectionId::new(1))
14419            .unwrap();
14420        let parked = transient.wait_for_select(|at| at.turn > turns).await;
14421        assert_eq!(
14422            parked.wake_after, None,
14423            "transient HELLO must cancel the deadline"
14424        );
14425        tokio::time::advance(Duration::from_secs(300)).await;
14426        tokio::task::yield_now().await;
14427        assert_eq!(transient.module.status().unwrap().pid, pid);
14428        assert!(transient.module.terminal_history().entries.is_empty());
14429        transient.module.stop().await.unwrap();
14430    }
14431
14432    #[tokio::test(start_paused = true)]
14433    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14434        let actor = Actor::start_with_policy(
14435            ModuleProtocol::Subc,
14436            Duration::from_secs(30),
14437            Duration::from_secs(10),
14438            RestartPolicy::new(1, Duration::ZERO),
14439        )
14440        .await;
14441        let generation = actor.module.status().unwrap().spawn_generation;
14442        actor
14443            .expire_registration_wait(Duration::from_secs(10))
14444            .await;
14445        actor.wait_for_select(|at| at.generation > generation).await;
14446        actor
14447            .expire_registration_wait(Duration::from_secs(10))
14448            .await;
14449        let retry = wait_for("failed retry schedule", || {
14450            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14451            (state.state == ModuleState::Failed)
14452                .then_some(state.failed_retry.pending)
14453                .flatten()
14454        })
14455        .await;
14456        assert_eq!(retry.delay, Duration::from_secs(600));
14457        assert_eq!(retry.attempt, 1);
14458        let entries = actor.module.terminal_history().entries;
14459        assert_eq!(entries.len(), 2);
14460        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14461        assert!(entries[1]
14462            .disposition_detail
14463            .as_deref()
14464            .unwrap()
14465            .contains("registration_timeout"));
14466        tokio::time::advance(Duration::from_secs(599)).await;
14467        tokio::task::yield_now().await;
14468        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14469        tokio::time::advance(Duration::from_secs(1)).await;
14470        actor
14471            .wait_for_select(|at| at.generation > generation + 1)
14472            .await;
14473        // The slow retry is another ordinary spawn, with its own HELLO budget.
14474        actor
14475            .expire_registration_wait(Duration::from_secs(10))
14476            .await;
14477        wait_for("slow retry registration timeout", || {
14478            (actor.module.terminal_history().entries.len() == 3).then_some(())
14479        })
14480        .await;
14481        actor.module.stop().await.unwrap();
14482    }
14483
14484    #[tokio::test(start_paused = true)]
14485    async fn registration_timeout_honours_ten_second_override() {
14486        let actor = Actor::start_with_policy(
14487            ModuleProtocol::Subc,
14488            Duration::from_secs(30),
14489            Duration::from_secs(10),
14490            RestartPolicy::new(3, Duration::ZERO),
14491        )
14492        .await;
14493        let generation = actor.module.status().unwrap().spawn_generation;
14494        tokio::time::advance(Duration::from_secs(9)).await;
14495        tokio::task::yield_now().await;
14496        assert!(actor.module.terminal_history().entries.is_empty());
14497        actor.expire_registration_wait(Duration::from_secs(1)).await;
14498        actor.wait_for_select(|at| at.generation > generation).await;
14499        assert_eq!(actor.module.terminal_history().entries.len(), 1);
14500        actor.module.stop().await.unwrap();
14501    }
14502
14503    #[tokio::test]
14504    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14505        use std::os::unix::process::{CommandExt, ExitStatusExt};
14506
14507        struct Helper(std::process::Child);
14508        impl Drop for Helper {
14509            fn drop(&mut self) {
14510                // This handle still owns the helper until wait returns. Never
14511                // clean up its old process group after the leader is reaped.
14512                if matches!(self.0.try_wait(), Ok(None)) {
14513                    let _ = self.0.kill();
14514                    let _ = self.0.wait();
14515                }
14516            }
14517        }
14518
14519        let home = ScratchDir::new("registration-timeout-helper");
14520        let supervisor = Supervisor::new_for_test(
14521            Arc::new(Registry::default()),
14522            RestartPolicy::new(0, Duration::ZERO),
14523        );
14524        let spec = ModuleSpec {
14525            module_id: "registration-timeout-group".into(),
14526            program: "/bin/sleep".into(),
14527            args: vec!["120".into()],
14528            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14529                .into_iter()
14530                .map(|key| (key.into(), home.path().display().to_string()))
14531                .collect(),
14532            reserved: false,
14533            reserved_prefixes: vec![],
14534            protocol: ModuleProtocol::Subc,
14535            overlap: Default::default(),
14536        };
14537        let module = supervisor
14538            .supervise_configured_requested(
14539                spec,
14540                true,
14541                HealthConfig::default(),
14542                None,
14543                RestartPolicy::new(0, Duration::ZERO),
14544                Duration::from_secs(10),
14545                None,
14546            )
14547            .unwrap();
14548        let leader = timeout(Duration::from_secs(5), async {
14549            loop {
14550                if let Some(pid) = module.status().unwrap().pid {
14551                    break pid;
14552                }
14553                sleep(Duration::from_millis(5)).await;
14554            }
14555        })
14556        .await
14557        .expect("supervised group leader must be reported before the deadline");
14558        assert!(leader > 1);
14559        let ready = home.join("helper.ready");
14560        // Own the helper separately so even a broken group kill can be cleaned
14561        // up by its own unreaped child handle, without signalling a stale pgid.
14562        let mut helper = Helper(
14563            std::process::Command::new("/bin/sh")
14564                .args([
14565                    "-c",
14566                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14567                    "helper",
14568                ])
14569                .arg(&ready)
14570                .process_group(i32::try_from(leader).unwrap())
14571                .stdout(std::process::Stdio::null())
14572                .stderr(std::process::Stdio::null())
14573                .spawn()
14574                .unwrap(),
14575        );
14576        timeout(Duration::from_secs(5), async {
14577            while !ready.exists() {
14578                assert!(helper.0.try_wait().unwrap().is_none());
14579                sleep(Duration::from_millis(5)).await;
14580            }
14581        })
14582        .await
14583        .expect("helper must install TERM ignore before the registration deadline");
14584        let helper_pid =
14585            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14586        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14587        sleep(Duration::from_millis(20)).await;
14588        assert!(
14589            helper.0.try_wait().unwrap().is_none(),
14590            "helper must ignore TERM"
14591        );
14592
14593        timeout(Duration::from_secs(20), async {
14594            while module.state().unwrap() != ModuleState::Failed {
14595                sleep(Duration::from_millis(5)).await;
14596            }
14597        })
14598        .await
14599        .expect("registration timeout must stop and reap the group leader");
14600        let helper_exit = timeout(Duration::from_secs(3), async {
14601            loop {
14602                if let Some(status) = helper.0.try_wait().unwrap() {
14603                    break status;
14604                }
14605                sleep(Duration::from_millis(5)).await;
14606            }
14607        })
14608        .await
14609        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14610        assert_eq!(helper_exit.signal(), Some(9));
14611        assert!(!cortexkit_test_support::process_alive(
14612            i32::try_from(leader).unwrap()
14613        ));
14614        assert_eq!(
14615            module.status().unwrap().last_exit.unwrap().kind,
14616            ExitKind::Crash
14617        );
14618        assert!(module.terminal_history().entries[0]
14619            .disposition_detail
14620            .as_deref()
14621            .unwrap()
14622            .contains("registration_timeout"));
14623        module.stop().await.unwrap();
14624    }
14625
14626    #[tokio::test(start_paused = true)]
14627    async fn registration_timeout_does_not_count_an_earlier_exit() {
14628        let actor = Actor::start_with_policy(
14629            ModuleProtocol::Subc,
14630            Duration::from_secs(30),
14631            DEFAULT_REGISTER_TIMEOUT,
14632            RestartPolicy::new(0, Duration::ZERO),
14633        )
14634        .await;
14635        let pid = actor.module.status().unwrap().pid.unwrap();
14636        rustix::process::kill_process(
14637            rustix::process::Pid::from_raw(pid as i32).unwrap(),
14638            rustix::process::Signal::TERM,
14639        )
14640        .unwrap();
14641        wait_for("early exit", || {
14642            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14643        })
14644        .await;
14645        tokio::time::advance(Duration::from_secs(121)).await;
14646        tokio::task::yield_now().await;
14647        let entries = actor.module.terminal_history().entries;
14648        assert_eq!(entries.len(), 1);
14649        assert!(!entries[0]
14650            .disposition_detail
14651            .as_deref()
14652            .unwrap()
14653            .contains("registration_timeout"));
14654        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14655        actor.module.stop().await.unwrap();
14656    }
14657
14658    #[tokio::test(start_paused = true)]
14659    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14660        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14661        let pid = actor.module.status().unwrap().pid;
14662        tokio::time::advance(Duration::from_secs(119)).await;
14663        actor.roster.close();
14664        let turns = actor.turns();
14665        tokio::time::advance(Duration::from_secs(300)).await;
14666        let parked = actor.wait_for_select(|at| at.turn > turns).await;
14667        assert_eq!(parked.wake_after, None);
14668        assert_eq!(actor.module.status().unwrap().pid, pid);
14669        assert!(actor.module.terminal_history().entries.is_empty());
14670        actor.module.stop().await.unwrap();
14671        let status = actor.module.status().unwrap();
14672        assert_eq!(status.restart_count, 0);
14673        assert_eq!(
14674            actor.module.terminal_history().entries[0].disposition,
14675            TerminalDisposition::DaemonShutdown
14676        );
14677    }
14678
14679    #[tokio::test(start_paused = true)]
14680    async fn no_health_children_park_for_sixty_seconds() {
14681        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14682        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14683        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14684        unadvertised.hello(1, false);
14685        // Include any one-off registration wake in the budget; receiving a
14686        // no-health HELLO must not turn a parked child into a polling child.
14687        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
14688        // a single 60 s jump would only observe one expired timer.
14689        for _ in 0..6000 {
14690            tokio::time::advance(Duration::from_millis(10)).await;
14691            tokio::task::yield_now().await;
14692        }
14693        for actor in [&none, &unregistered, &unadvertised] {
14694            assert!(
14695                actor.turns() <= 5,
14696                "no-health actor woke {} times",
14697                actor.turns()
14698            );
14699            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14700        }
14701    }
14702
14703    #[tokio::test(start_paused = true)]
14704    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14705        // With a zero cadence the first probe is due at once, so this measures
14706        // only how fast the registration notification starts health probing,
14707        // not the normal 30-33 s wait before a first probe (checked below).
14708        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14709        let tick = Instant::now();
14710        actor.hello(1, true);
14711        actor.probe().await;
14712        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14713    }
14714
14715    #[tokio::test(start_paused = true)]
14716    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14717        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14718        let hello_tick = Instant::now();
14719        actor.hello(1, true);
14720        let armed = actor
14721            .wait_for_select(|checkpoint| {
14722                checkpoint.registered_connection == Some(ConnectionId::new(1))
14723                    && checkpoint.next_probe_at.is_some()
14724            })
14725            .await;
14726        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14727        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14728        let first_deadline = hello_tick + delay;
14729        assert_eq!(armed.next_probe_at, Some(first_deadline));
14730        tokio::time::advance(Duration::from_secs(29)).await;
14731        let turns = actor.turns();
14732        actor
14733            .registry
14734            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14735            .unwrap();
14736        let updated = actor
14737            .wait_for_select(|checkpoint| checkpoint.turn > turns)
14738            .await;
14739        assert_eq!(
14740            updated.next_probe_at,
14741            Some(first_deadline),
14742            "catalog update must not reset cadence"
14743        );
14744        actor.no_probe();
14745        tokio::time::advance(first_deadline - Instant::now()).await;
14746        let frame = actor.probe().await;
14747        actor.answer(1, frame);
14748        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14749        let next_deadline = Instant::now() + next;
14750        let rearmed = actor
14751            .wait_for_select(|checkpoint| {
14752                checkpoint
14753                    .next_probe_at
14754                    .is_some_and(|deadline| deadline > first_deadline)
14755            })
14756            .await;
14757        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14758        tokio::time::advance(next - Duration::from_nanos(1)).await;
14759        actor.no_probe();
14760        tokio::time::advance(Duration::from_nanos(1)).await;
14761        actor.probe().await;
14762    }
14763
14764    #[tokio::test(start_paused = true)]
14765    async fn disconnect_disarms_and_reconnect_rearms_health() {
14766        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14767        actor.hello(1, true);
14768        actor
14769            .wait_for_select(|checkpoint| {
14770                checkpoint.registered_connection == Some(ConnectionId::new(1))
14771                    && checkpoint.next_probe_at.is_some()
14772            })
14773            .await;
14774        let turns = actor.turns();
14775        let tick = Instant::now();
14776        actor
14777            .registry
14778            .deregister_connection(ConnectionId::new(1))
14779            .unwrap();
14780        let disarmed = actor
14781            .wait_for_select(|checkpoint| {
14782                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14783            })
14784            .await;
14785        assert_eq!(disarmed.next_probe_at, None);
14786        assert_eq!(disarmed.wake_after, None);
14787        assert_eq!(
14788            Instant::now(),
14789            tick,
14790            "disconnect must disarm in the notification tick"
14791        );
14792        tokio::time::advance(Duration::from_secs(60)).await;
14793        actor.no_probe();
14794        let reconnect_tick = Instant::now();
14795        actor.hello(2, true);
14796        let rearmed = actor
14797            .wait_for_select(|checkpoint| {
14798                checkpoint.registered_connection == Some(ConnectionId::new(2))
14799                    && checkpoint.next_probe_at.is_some()
14800            })
14801            .await;
14802        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14803        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14804        tokio::time::advance(delay).await;
14805        actor.probe().await;
14806    }
14807
14808    #[tokio::test(start_paused = true)]
14809    async fn reregistration_without_health_disarms_the_old_schedule() {
14810        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14811        actor.hello(1, true);
14812        actor
14813            .wait_for_select(|checkpoint| {
14814                checkpoint.registered_connection == Some(ConnectionId::new(1))
14815                    && checkpoint.next_probe_at.is_some()
14816            })
14817            .await;
14818        actor
14819            .registry
14820            .deregister_connection(ConnectionId::new(1))
14821            .unwrap();
14822        actor.hello(2, false);
14823        let disarmed = actor
14824            .wait_for_select(|checkpoint| {
14825                checkpoint.registered_connection == Some(ConnectionId::new(2))
14826            })
14827            .await;
14828        assert_eq!(disarmed.next_probe_at, None);
14829        assert_eq!(disarmed.wake_after, None);
14830        tokio::time::advance(Duration::from_secs(60)).await;
14831        actor.no_probe();
14832    }
14833
14834    #[tokio::test(start_paused = true)]
14835    async fn restart_waits_for_its_own_hello_before_rearming_health() {
14836        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14837        actor.hello(1, true);
14838        let armed = actor
14839            .wait_for_select(|checkpoint| {
14840                checkpoint.registered_connection == Some(ConnectionId::new(1))
14841                    && checkpoint.next_probe_at.is_some()
14842            })
14843            .await;
14844        // Simulate the old process's connection closing before the restart
14845        // tears the process down. The replacement process has started but has
14846        // not sent its HELLO, so it is not registered.
14847        actor
14848            .registry
14849            .deregister_connection(ConnectionId::new(1))
14850            .unwrap();
14851        actor.module.restart(Some(0)).await.unwrap();
14852        let parked = actor
14853            .wait_for_select(|checkpoint| {
14854                checkpoint.generation > armed.generation
14855                    && checkpoint.registered_connection.is_none()
14856                    && checkpoint.next_probe_at.is_none()
14857            })
14858            .await;
14859        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
14860        actor.rx.close();
14861        // Drain the stop notice the restart queued for the old process's connection,
14862        // so it isn't mistaken for traffic to the replacement.
14863        while actor.rx.try_recv().is_ok() {}
14864        let turns = actor.turns();
14865        tokio::time::advance(Duration::from_secs(60)).await;
14866        let after = actor.turns();
14867        eprintln!(
14868            "restart park turns: before={turns} after={after} delta={}",
14869            after - turns
14870        );
14871        assert_eq!(after, turns, "replacement must park before HELLO");
14872        let hello_tick = Instant::now();
14873        actor.hello(2, true);
14874        let rearmed = actor
14875            .wait_for_select(|checkpoint| {
14876                checkpoint.registered_connection == Some(ConnectionId::new(2))
14877                    && checkpoint.next_probe_at.is_some()
14878            })
14879            .await;
14880        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14881        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14882        tokio::time::advance(delay).await;
14883        actor.probe().await;
14884    }
14885
14886    #[tokio::test(start_paused = true)]
14887    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14888        use tokio::{
14889            io::{AsyncReadExt, AsyncWriteExt},
14890            net::TcpListener,
14891        };
14892        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14893        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14894        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14895        let (started_tx, mut started_rx) = mpsc::channel(8);
14896        let peer = tokio::spawn(async move {
14897            loop {
14898                let (mut socket, _) = listener.accept().await.unwrap();
14899                let mut request = [0; 1024];
14900                assert!(socket.read(&mut request).await.unwrap() > 0);
14901                started_tx.send(Instant::now()).await.unwrap();
14902                socket
14903                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14904                    .await
14905                    .unwrap();
14906            }
14907        });
14908        actor
14909            .module
14910            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14911            .await
14912            .unwrap();
14913        let first_tick = Instant::now();
14914        let first = actor
14915            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14916            .await;
14917        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14918        assert!(started_rx.try_recv().is_err());
14919
14920        // Restart just before the old deadline: reusing it would give the new
14921        // process only one second to start, rather than a full cadence.
14922        tokio::time::advance(
14923            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14924        )
14925        .await;
14926        actor.module.restart(Some(0)).await.unwrap();
14927        let spawn_tick = Instant::now();
14928        let replacement = actor
14929            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14930            .await;
14931        let deadline = replacement.next_probe_at.unwrap();
14932        assert!(
14933            deadline >= spawn_tick + actor.health.cadence,
14934            "a replacement must not inherit its predecessor's probe deadline"
14935        );
14936        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14937        assert!(
14938            started_rx.try_recv().is_err(),
14939            "no HTTP probe during startup grace"
14940        );
14941        tokio::time::advance(deadline - Instant::now()).await;
14942        let landed = wait_for("first replacement HTTP probe", || {
14943            started_rx.try_recv().ok()
14944        })
14945        .await;
14946        assert!(landed >= spawn_tick + actor.health.cadence);
14947        wait_for("healthy replacement", || {
14948            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14949                .then_some(())
14950        })
14951        .await;
14952        peer.abort();
14953    }
14954
14955    #[tokio::test(start_paused = true)]
14956    async fn rescan_adds_and_removes_http_health_without_polling() {
14957        use tokio::{
14958            io::{AsyncReadExt, AsyncWriteExt},
14959            net::TcpListener,
14960        };
14961        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14962        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14963        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14964        // One millisecond separates successive probes without a polling delay.
14965        actor.health.cadence = Duration::from_millis(1);
14966        let (started_tx, mut started_rx) = mpsc::channel(8);
14967        let peer = tokio::spawn(async move {
14968            loop {
14969                let (mut socket, _) = listener.accept().await.unwrap();
14970                let mut request = [0; 1024];
14971                assert!(socket.read(&mut request).await.unwrap() > 0);
14972                started_tx.send(()).await.unwrap();
14973                socket
14974                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14975                    .await
14976                    .unwrap();
14977            }
14978        });
14979        let tick = Instant::now();
14980        let turns = actor.turns();
14981        actor
14982            .module
14983            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14984            .await
14985            .unwrap();
14986        let armed = actor
14987            .wait_for_select(|checkpoint| {
14988                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14989            })
14990            .await;
14991        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14992        assert_eq!(Instant::now(), tick);
14993        tokio::time::advance(Duration::from_millis(1)).await;
14994        let mut started = false;
14995        wait_for("successful HTTP probe", || {
14996            started |= started_rx.try_recv().is_ok();
14997            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14998                .then_some(())
14999        })
15000        .await;
15001        assert!(started, "added HTTP check must start at its first deadline");
15002        assert_eq!(
15003            actor.module.status().unwrap().health.status,
15004            SupervisorHealthStatus::Ok
15005        );
15006        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15007        actor.health.http = None;
15008        let turns = actor.turns();
15009        actor
15010            .module
15011            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15012            .await
15013            .unwrap();
15014        let parked = actor
15015            .wait_for_select(|checkpoint| {
15016                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15017            })
15018            .await;
15019        assert_eq!(parked.wake_after, None);
15020        let turns = actor.turns();
15021        tokio::time::advance(Duration::from_secs(60)).await;
15022        assert_eq!(
15023            actor.turns(),
15024            turns,
15025            "removed HTTP check must park the actor"
15026        );
15027        assert!(
15028            started_rx.try_recv().is_err(),
15029            "removed HTTP check must stay disarmed"
15030        );
15031        assert_eq!(
15032            actor.module.status().unwrap().health,
15033            ModuleHealthStatus::default()
15034        );
15035        peer.abort();
15036    }
15037}
15038
15039#[cfg(test)]
15040mod drain_busy_reprobe_tests {
15041    //! The drain's busy-gauge wait, driven against an in-process module
15042    //! connection whose health answers the test scripts. Time is paused, so
15043    //! every elapsed duration below is exact virtual time.
15044    use super::*;
15045    use std::collections::BTreeMap;
15046    use std::sync::Mutex as StdMutex;
15047    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15048
15049    const MODULE_ID: &str = "drain-reprobe-module";
15050    const CONNECTION: u64 = 7;
15051    const GAUGE: &str = "runs_in_flight";
15052    /// The production default for both the health cadence and the drain
15053    /// ceiling, which is the pairing that used to cost a full ceiling.
15054    const CADENCE: Duration = Duration::from_secs(30);
15055    const CEILING: Duration = Duration::from_secs(30);
15056
15057    /// How the scripted module answers each `health.check`.
15058    #[derive(Clone, Copy)]
15059    enum Answer {
15060        /// Report the gauge busy until this instant, quiet from then on, after
15061        /// taking `latency` to answer.
15062        BusyUntil { until: Instant, latency: Duration },
15063        /// Answer at once, with metrics that leave the declared gauge out.
15064        OmitsGauge,
15065        /// Never answer.
15066        Silent,
15067    }
15068
15069    struct Drain {
15070        forwarding: Arc<ForwardingTable>,
15071        runtime: SupervisorRuntimeConfig,
15072        endpoint: crate::ModuleEndpointId,
15073        /// When each probe reached the module.
15074        probes: Arc<StdMutex<Vec<Instant>>>,
15075        responder: tokio::task::JoinHandle<()>,
15076    }
15077
15078    impl Drain {
15079        fn start(answer: Answer) -> Self {
15080            let registry = Arc::new(Registry::default());
15081            let forwarding = Arc::new(ForwardingTable::default());
15082            let health = HealthConfig {
15083                cadence: CADENCE,
15084                ..HealthConfig::default()
15085            };
15086            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15087                .with_forwarding(forwarding.clone())
15088                .with_health_config(health)
15089                .runtime_config();
15090            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15091            let endpoint = forwarding
15092                .register_module_connection(
15093                    ConnectionId::new(CONNECTION),
15094                    MODULE_ID.to_string(),
15095                    subc_protocol::PROTOCOL_VERSION,
15096                    Concurrency::ModuleManaged,
15097                    FrameSink::new(tx),
15098                )
15099                .unwrap();
15100            let probes = Arc::new(StdMutex::new(Vec::new()));
15101            let responder = tokio::spawn({
15102                let forwarding = forwarding.clone();
15103                let probes = probes.clone();
15104                async move {
15105                    while let Some(frame) = rx.recv().await {
15106                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15107                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15108                        probes.lock().unwrap().push(Instant::now());
15109                        let metrics = match answer {
15110                            Answer::BusyUntil { until, latency } => {
15111                                sleep(latency).await;
15112                                let busy = u64::from(Instant::now() < until);
15113                                serde_json::json!({ GAUGE: busy })
15114                            }
15115                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15116                            Answer::Silent => continue,
15117                        };
15118                        // A probe abandoned at its deadline has no RPC left to
15119                        // complete, so a late answer may be refused.
15120                        let _ = forwarding.complete_module_control_rpc(
15121                            ConnectionId::new(CONNECTION),
15122                            frame.header.corr,
15123                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15124                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15125                                status: HealthStatus::Ok,
15126                                detail: None,
15127                                metrics: Some(metrics),
15128                            }),
15129                        );
15130                    }
15131                }
15132            });
15133            Self {
15134                forwarding,
15135                runtime,
15136                endpoint,
15137                probes,
15138                responder,
15139            }
15140        }
15141
15142        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15143        /// how long it took.
15144        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15145            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15146            (drained, elapsed)
15147        }
15148
15149        /// [`Self::run`], also returning what the drain's probes read.
15150        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15151            let started = Instant::now();
15152            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15153            let drained = wait_for_forwarding_quiescence(
15154                &self.forwarding,
15155                MODULE_ID,
15156                &self.runtime,
15157                self.endpoint,
15158                started + ceiling,
15159                &[GAUGE.to_string()],
15160                DrainScope::Active,
15161                &mut trace,
15162            )
15163            .await
15164            .unwrap();
15165            (drained, started.elapsed(), trace)
15166        }
15167
15168        fn probes(&self) -> Vec<Instant> {
15169            self.probes.lock().unwrap().clone()
15170        }
15171    }
15172
15173    impl Drop for Drain {
15174        fn drop(&mut self) {
15175            self.responder.abort();
15176        }
15177    }
15178
15179    #[tokio::test(start_paused = true)]
15180    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15181        let drain = Drain::start(Answer::BusyUntil {
15182            until: Instant::now() + Duration::from_secs(1),
15183            latency: Duration::ZERO,
15184        });
15185        let (drained, elapsed) = drain.run(CEILING).await;
15186        assert!(
15187            drained,
15188            "gauges that read zero during the drain must let it finish drained, \
15189             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15190        );
15191        assert!(
15192            elapsed < Duration::from_secs(3),
15193            "drain took {elapsed:?} after the gauges went quiet at 1s"
15194        );
15195        let probes = drain.probes();
15196        assert!(
15197            probes.len() >= 2,
15198            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15199        );
15200    }
15201
15202    /// The same shape in real time on a multi-threaded runtime: the first
15203    /// probe is answered busy at once, within a millisecond of the drain
15204    /// starting, and later probes read idle. The drain must keep probing
15205    /// after that first instant answer and end soon after the gauge falls.
15206    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15207    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15208        let drain = Drain::start(Answer::BusyUntil {
15209            until: Instant::now() + Duration::from_millis(600),
15210            latency: Duration::ZERO,
15211        });
15212        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15213        assert!(drained, "the gauge fell at 600ms: {trace}");
15214        assert!(
15215            elapsed < Duration::from_secs(3),
15216            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15217        );
15218        let probes = drain.probes();
15219        assert!(
15220            probes.len() >= 2 && trace.probes as usize == probes.len(),
15221            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15222        );
15223    }
15224
15225    #[tokio::test(start_paused = true)]
15226    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15227        let started = Instant::now();
15228        let drain = Drain::start(Answer::BusyUntil {
15229            until: started + Duration::from_secs(3600),
15230            latency: Duration::ZERO,
15231        });
15232        let (drained, elapsed) = drain.run(CEILING).await;
15233        assert!(!drained, "a module busy throughout must not drain");
15234        assert!(
15235            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15236            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15237        );
15238        // Re-probing is paced by the interval, not a busy loop: at most one
15239        // probe per interval across the ceiling, plus the first at time zero.
15240        let probes = drain.probes();
15241        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15242        assert!(
15243            probes.len() as u128 <= max_probes,
15244            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15245            probes.len()
15246        );
15247        for pair in probes.windows(2) {
15248            assert!(
15249                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15250                "consecutive probes {:?} apart",
15251                pair[1] - pair[0]
15252            );
15253        }
15254    }
15255
15256    #[tokio::test(start_paused = true)]
15257    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15258        // Each answer takes longer than the re-probe interval; the next probe
15259        // must wait for the previous answer and then the interval.
15260        let latency = Duration::from_secs(1);
15261        let drain = Drain::start(Answer::BusyUntil {
15262            until: Instant::now() + Duration::from_secs(3600),
15263            latency,
15264        });
15265        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15266        assert!(!drained);
15267        let probes = drain.probes();
15268        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15269        for pair in probes.windows(2) {
15270            assert!(
15271                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15272                "a probe started {:?} after the previous one, before it could finish",
15273                pair[1] - pair[0]
15274            );
15275        }
15276    }
15277
15278    #[tokio::test(start_paused = true)]
15279    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15280        let drain = Drain::start(Answer::Silent);
15281        let ceiling = Duration::from_secs(2);
15282        let (drained, elapsed) = drain.run(ceiling).await;
15283        assert!(!drained, "an unanswered probe counts as busy");
15284        assert!(
15285            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15286            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15287        );
15288        assert_eq!(
15289            drain.probes().len(),
15290            1,
15291            "the unanswered probe holds the only slot until the ceiling"
15292        );
15293    }
15294
15295    /// Every event logged while it is installed, as `message` plus fields.
15296    #[derive(Clone, Default)]
15297    struct LogCapture {
15298        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15299    }
15300
15301    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15302        fn on_event(
15303            &self,
15304            event: &tracing::Event<'_>,
15305            _context: tracing_subscriber::layer::Context<'_, S>,
15306        ) {
15307            struct Fields(BTreeMap<String, String>);
15308            impl tracing::field::Visit for Fields {
15309                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15310                    self.0
15311                        .insert(field.name().to_string(), format!("{value:?}"));
15312                }
15313            }
15314            let mut fields = Fields(BTreeMap::new());
15315            event.record(&mut fields);
15316            self.events.lock().unwrap().push(fields.0);
15317        }
15318    }
15319
15320    impl LogCapture {
15321        fn install(&self) -> tracing::subscriber::DefaultGuard {
15322            use tracing_subscriber::layer::SubscriberExt;
15323            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15324        }
15325
15326        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15327            self.events
15328                .lock()
15329                .unwrap()
15330                .iter()
15331                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15332                .cloned()
15333                .collect()
15334        }
15335    }
15336
15337    /// A module that declares a busy gauge but leaves it out of its health
15338    /// answers holds a drain to its ceiling. The timeout line must say so by
15339    /// name, and the drain must warn about the missing gauge once, not on
15340    /// every probe.
15341    #[tokio::test(start_paused = true)]
15342    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15343        let capture = LogCapture::default();
15344        let _guard = capture.install();
15345        let drain = Drain::start(Answer::OmitsGauge);
15346        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15347        assert!(!drained, "a gauge never reported cannot read quiet");
15348        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15349
15350        let missing = capture.with_message("lacks a declared busy gauge");
15351        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15352        assert_eq!(
15353            missing[0].get("missing_gauges").map(String::as_str),
15354            Some(GAUGE)
15355        );
15356
15357        warn_drain_timed_out(
15358            MODULE_ID,
15359            Duration::from_secs(2),
15360            RouteCloseReason::Restart,
15361            &Default::default(),
15362            &trace,
15363        );
15364        let timeouts = capture.with_message("route drain timed out");
15365        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15366        let gauges = &timeouts[0]["busy_gauges"];
15367        assert!(
15368            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15369            "the timeout line must name the gauge and why it read busy: {gauges}"
15370        );
15371        assert!(
15372            gauges.ends_with("; no value ever read"),
15373            "no value was ever read for the gauge: {gauges}"
15374        );
15375        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15376    }
15377
15378    /// The trace names a busy value and when it was read. For a probe that
15379    /// never got an answer, it names the probe's error kind and says that no
15380    /// value was ever read.
15381    #[tokio::test(start_paused = true)]
15382    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15383        let busy = Drain::start(Answer::BusyUntil {
15384            until: Instant::now() + Duration::from_secs(3600),
15385            latency: Duration::ZERO,
15386        });
15387        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15388        let rendered = trace.to_string();
15389        assert!(
15390            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15391                && rendered.contains("; last value non_zero(1) @"),
15392            "{rendered}"
15393        );
15394
15395        let silent = Drain::start(Answer::Silent);
15396        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15397        // The probe is capped at the 1 s ceiling, and its reading is stamped
15398        // when it gave up.
15399        assert_eq!(
15400            trace.to_string(),
15401            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15402        );
15403        assert_eq!(trace.probes, 1);
15404    }
15405
15406    /// A gauge value that is not a count reads busy and is shown as sent.
15407    #[test]
15408    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15409        let gauges = [GAUGE.to_string()];
15410        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15411        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15412        assert_eq!(
15413            busy_gauge_observation(&verdicts),
15414            BusyGaugeObservation::Busy
15415        );
15416        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15417        assert_eq!(
15418            busy_gauge_observation(&zero),
15419            BusyGaugeObservation::Quiescent
15420        );
15421        let none = gauge_verdicts(None, &gauges);
15422        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15423    }
15424}
15425
15426#[cfg(test)]
15427mod health_evidence_tests {
15428    use super::{HealthProbeError, HealthProbeEvidence};
15429    use std::collections::HashSet;
15430
15431    /// The evidential asymmetry, asserted rather than described.
15432    ///
15433    /// Exactly ONE observation is proof a module cannot serve, and the one that
15434    /// fires under CPU starvation is not it. Before the split, all fifteen
15435    /// construction sites collapsed into a single String, so a timeout carried the
15436    /// same weight as a dead lane -- which is how a healthy module was restarted
15437    /// three times in one day.
15438    #[test]
15439    fn only_a_dead_lane_is_proof_of_death() {
15440        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15441        // Three non-proof classes, each for a different reason: silence is
15442        // consistent with health, a bad answer proves the module ALIVE, and a
15443        // daemon-side fault never reached the module at all.
15444        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15445        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15446        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15447    }
15448
15449    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
15450    ///
15451    /// A shared label renders two different observations identically in the line an
15452    /// operator reads after an unexplained restart -- the exact confusion this
15453    /// change removes.
15454    #[test]
15455    fn every_evidence_class_has_a_distinct_label() {
15456        let labels = [
15457            HealthProbeError::lane_dead("").label(),
15458            HealthProbeError::no_answer("").label(),
15459            HealthProbeError::bad_answer("").label(),
15460            HealthProbeError::misconfigured("").label(),
15461        ];
15462        let unique: HashSet<_> = labels.iter().collect();
15463        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15464    }
15465
15466    /// The class is additional information, not a replacement.
15467    ///
15468    /// An operator needs both "this was silence" and the specific text saying how
15469    /// long we waited; a classification that swallowed the message would trade one
15470    /// missing distinction for another.
15471    #[test]
15472    fn classification_preserves_the_original_message() {
15473        let err = HealthProbeError::no_answer("module did not answer within 5s");
15474        assert_eq!(err.to_string(), "module did not answer within 5s");
15475        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15476    }
15477}
15478
15479#[cfg(test)]
15480mod health_tombstone_tests {
15481    use std::{path::PathBuf, sync::Arc, time::Duration};
15482
15483    use subc_protocol::{
15484        manifest::Concurrency,
15485        session::{HealthStatus, ModuleControlResponse},
15486    };
15487    use tokio::sync::mpsc;
15488
15489    use super::{
15490        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15491        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15492    };
15493    use crate::{
15494        control::ControlHandler,
15495        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15496        registry::{ConnectionId, Registry},
15497        router::FrameSink,
15498    };
15499
15500    struct ProbeHarness {
15501        spec: ModuleSpec,
15502        runtime: SupervisorRuntimeConfig,
15503        forwarding: Arc<ForwardingTable>,
15504        module_connection: ConnectionId,
15505        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15506        handler: ControlHandler,
15507        module: super::SupervisedModule,
15508    }
15509
15510    fn probe_harness() -> ProbeHarness {
15511        let registry = Arc::new(Registry::default());
15512        let forwarding = Arc::new(ForwardingTable::default());
15513        let supervisor_handle = super::SupervisorHandle::new();
15514        let health = HealthConfig {
15515            http: None,
15516            cadence: Duration::from_secs(30),
15517            deadline: Duration::from_secs(5),
15518            failure_threshold: 3,
15519            on_degraded: HealthAction::Report,
15520            on_failing: HealthAction::Report,
15521            critical: false,
15522        };
15523        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15524            .with_forwarding(Arc::clone(&forwarding))
15525            .with_handle(supervisor_handle.clone())
15526            .with_health_config(health);
15527        let spec = ModuleSpec {
15528            module_id: "late-health-module".to_string(),
15529            program: PathBuf::from("disabled-module"),
15530            args: Vec::new(),
15531            env: Vec::new(),
15532            reserved: false,
15533            reserved_prefixes: Vec::new(),
15534            protocol: ModuleProtocol::Subc,
15535            overlap: Default::default(),
15536        };
15537        let module = supervisor
15538            .supervise_configured(spec.clone(), false)
15539            .unwrap();
15540        let runtime = supervisor.runtime_config();
15541        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15542            .with_supervisor(supervisor_handle);
15543        let module_connection = ConnectionId::new(700);
15544        let (module_tx, module_rx) = mpsc::channel(8);
15545        forwarding
15546            .register_module_connection(
15547                module_connection,
15548                spec.module_id.clone(),
15549                subc_protocol::PROTOCOL_VERSION,
15550                Concurrency::ModuleManaged,
15551                FrameSink::new(module_tx),
15552            )
15553            .unwrap();
15554
15555        ProbeHarness {
15556            spec,
15557            runtime,
15558            forwarding,
15559            module_connection,
15560            module_rx,
15561            handler,
15562            module,
15563        }
15564    }
15565
15566    async fn finish_after(
15567        harness: &mut ProbeHarness,
15568        stall: Duration,
15569    ) -> ModuleControlRpcCompletion {
15570        assert!(stall > harness.runtime.health.deadline);
15571        let deadline = harness.runtime.health.deadline;
15572        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15573        let answer = async {
15574            let frame = harness.module_rx.recv().await.expect("health.check frame");
15575            tokio::time::advance(deadline).await;
15576            tokio::task::yield_now().await;
15577            tokio::time::advance(stall - deadline).await;
15578            harness
15579                .forwarding
15580                .complete_module_control_rpc(
15581                    harness.module_connection,
15582                    frame.header.corr,
15583                    Some("health.check"),
15584                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15585                        status: HealthStatus::Ok,
15586                        detail: None,
15587                        metrics: None,
15588                    }),
15589                )
15590                .unwrap()
15591        };
15592        let (probe_result, completion) = tokio::join!(probe, answer);
15593        let err = probe_result.expect_err("probe must miss its deadline");
15594        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15595        completion
15596    }
15597
15598    async fn time_out_without_answer(harness: &mut ProbeHarness) {
15599        let deadline = harness.runtime.health.deadline;
15600        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15601        let exhaust_deadline = async {
15602            let _frame = harness.module_rx.recv().await.expect("health.check frame");
15603            tokio::time::advance(deadline).await;
15604            tokio::task::yield_now().await;
15605        };
15606        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15607        let err = probe_result.expect_err("probe must miss its deadline");
15608        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15609    }
15610
15611    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15612        let registry = Arc::clone(&harness.module.inner.registry);
15613        let snapshot = Arc::clone(&harness.module.inner.snapshot);
15614        let process_liveness = super::SupervisorProcessLiveness::default();
15615        let mut child = None;
15616        let cycle = super::run_health_probe_cycle(
15617            &harness.spec,
15618            &harness.runtime,
15619            &registry,
15620            &process_liveness,
15621            &snapshot,
15622            &mut child,
15623        );
15624        let peer = async {
15625            let frame = harness.module_rx.recv().await.expect("health.check frame");
15626            if answer {
15627                harness
15628                    .forwarding
15629                    .complete_module_control_rpc(
15630                        harness.module_connection,
15631                        frame.header.corr,
15632                        Some("health.check"),
15633                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15634                            status: HealthStatus::Ok,
15635                            detail: None,
15636                            metrics: Some(serde_json::json!({"ready": true})),
15637                        }),
15638                    )
15639                    .unwrap();
15640            } else {
15641                tokio::time::advance(harness.runtime.health.deadline).await;
15642                tokio::task::yield_now().await;
15643            }
15644        };
15645        tokio::join!(cycle, peer);
15646    }
15647
15648    #[tokio::test(start_paused = true)]
15649    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15650        let mut harness = probe_harness();
15651        // Drive the probe cycle directly with an in-memory wire peer. Stop the
15652        // disabled module's monitor so only this test owns lifecycle transitions;
15653        // no OS process is launched, and a restart is observed at scheduling.
15654        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15655        monitor.abort();
15656        let _ = monitor.await;
15657        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15658            state.enabled = true;
15659            state.state = super::ModuleState::Running;
15660            state.process_alive = true;
15661        })
15662        .unwrap();
15663
15664        run_probe_cycle(&mut harness, true).await;
15665        assert_eq!(
15666            harness.module.status().unwrap().health.status,
15667            super::SupervisorHealthStatus::Ok
15668        );
15669
15670        for failures in 1..harness.runtime.health.failure_threshold {
15671            run_probe_cycle(&mut harness, false).await;
15672            let status = harness.module.status().unwrap();
15673            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15674            assert_eq!(status.health.consecutive_failures, failures);
15675            assert!(status.health.last_probe_ms.is_some());
15676            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15677            assert!(status.health.metrics.is_none());
15678            assert_eq!(status.state, super::ModuleState::Running);
15679            assert!(status.process_alive);
15680            assert_eq!(status.restart_count, 0);
15681            assert_eq!(status.lifetime_restarts, 0);
15682            assert!(status.health.last_action.is_none());
15683            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15684        }
15685
15686        run_probe_cycle(&mut harness, true).await;
15687        let recovered = harness.module.status().unwrap();
15688        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15689        assert_eq!(recovered.health.consecutive_failures, 0);
15690        assert!(recovered.health.detail.is_none());
15691        assert_eq!(
15692            recovered.health.metrics,
15693            Some(serde_json::json!({"ready": true}))
15694        );
15695        assert_eq!(recovered.lifetime_restarts, 0);
15696
15697        for failures in 1..=harness.runtime.health.failure_threshold {
15698            run_probe_cycle(&mut harness, false).await;
15699            let status = harness.module.status().unwrap();
15700            assert_eq!(status.health.consecutive_failures, failures);
15701            if failures < harness.runtime.health.failure_threshold {
15702                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15703                assert_eq!(status.state, super::ModuleState::Running);
15704                assert_eq!(status.lifetime_restarts, 0);
15705                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15706            } else {
15707                assert_eq!(
15708                    status.health.status,
15709                    super::SupervisorHealthStatus::Unresponsive
15710                );
15711                assert_eq!(status.state, super::ModuleState::Restarting);
15712                assert_eq!(status.restart_count, 1);
15713                assert_eq!(status.lifetime_restarts, 1);
15714                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15715                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15716            }
15717        }
15718    }
15719
15720    #[tokio::test(start_paused = true)]
15721    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15722        let mut harness = probe_harness();
15723
15724        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15725        let first_latency = match &first {
15726            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15727            other => panic!("late answer was not retained: {other:?}"),
15728        };
15729        assert!(harness.handler.observe_module_control_completion(first));
15730
15731        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15732        let second_latency = match &second {
15733            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15734            other => panic!("late answer was not retained: {other:?}"),
15735        };
15736        assert!(harness.handler.observe_module_control_completion(second));
15737
15738        assert_eq!(first_latency, Duration::from_secs(8));
15739        assert_eq!(
15740            second_latency - first_latency,
15741            Duration::from_secs(3),
15742            "latency must grow linearly with the additional stall"
15743        );
15744        let health = harness.module.status().unwrap().health;
15745        assert_eq!(health.late_answer_count, 2);
15746        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15747    }
15748
15749    /// A module that answers every probe late must never march to the kill
15750    /// threshold: the late answer proves it is alive, so it must clear the miss
15751    /// streak the timeout recorded. Without the reset, a CPU-starved module
15752    /// that serves every probe seconds past the deadline accumulates
15753    /// `consecutive_failures` to the threshold and is killed — the exact
15754    /// sequence from the 2026-08-14 aft disable, where the daemon logged
15755    /// "proves the module is alive" five times while counting five misses.
15756    #[tokio::test(start_paused = true)]
15757    async fn late_answer_clears_the_consecutive_failure_streak() {
15758        let mut harness = probe_harness();
15759
15760        // Timeout recorded first: the probe path saw no answer in time.
15761        time_out_without_answer(&mut harness).await;
15762        harness
15763            .module
15764            .record_health_probe_failure_for_test("[no-answer] test miss")
15765            .unwrap();
15766        assert_eq!(
15767            harness.module.status().unwrap().health.consecutive_failures,
15768            1,
15769            "precondition: the miss must be on the streak before the late answer"
15770        );
15771
15772        // The stalled reply then lands: proof of life.
15773        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15774        assert!(matches!(
15775            late,
15776            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15777        ));
15778        assert!(harness.handler.observe_module_control_completion(late));
15779
15780        let health = harness.module.status().unwrap().health;
15781        assert_eq!(
15782            health.consecutive_failures, 0,
15783            "a late answer is an answer: the streak must reset"
15784        );
15785        assert_eq!(health.late_answer_count, 1);
15786    }
15787
15788    #[tokio::test(start_paused = true)]
15789    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15790        let mut harness = probe_harness();
15791
15792        for _ in 0..20 {
15793            time_out_without_answer(&mut harness).await;
15794            assert_eq!(
15795                harness.forwarding.health_probe_tombstone_count().unwrap(),
15796                1
15797            );
15798        }
15799    }
15800}
15801
15802#[cfg(test)]
15803mod child_env_tests {
15804    #[cfg(windows)]
15805    use super::{
15806        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15807    };
15808    #[cfg(windows)]
15809    use std::sync::{Arc, OnceLock};
15810    #[cfg(windows)]
15811    #[test]
15812    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15813        let spec = spec(Vec::new());
15814        let mut command = Command::new("unused");
15815        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15816            .unwrap()
15817            .unwrap();
15818        let name = command
15819            .as_std()
15820            .get_envs()
15821            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15822            .and_then(|(_, value)| value)
15823            .unwrap();
15824        assert_eq!(name, OsStr::new(handoff.name()));
15825        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15826        assert!(command.as_std().get_envs().any(|(key, value)| key
15827            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15828            && value == Some(OsStr::new("env"))));
15829        assert!(command
15830            .as_std()
15831            .get_envs()
15832            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15833    }
15834
15835    #[cfg(windows)]
15836    #[test]
15837    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15838        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15839        snapshot.process_alive = true;
15840        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15841        assert_eq!(
15842            observed_launch_nonce_source(&snapshot).as_deref(),
15843            Some("env")
15844        );
15845        let consumed = Arc::new(OnceLock::new());
15846        snapshot.pipe_consumed = Some(consumed.clone());
15847        assert_eq!(
15848            observed_launch_nonce_source(&snapshot).as_deref(),
15849            Some("env")
15850        );
15851        consumed.set(()).unwrap();
15852        assert_eq!(
15853            observed_launch_nonce_source(&snapshot).as_deref(),
15854            Some("pipe")
15855        );
15856        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15857        assert_eq!(
15858            observed_launch_nonce_source(&snapshot).as_deref(),
15859            Some("env")
15860        );
15861        clear_current_process_facts(&mut snapshot);
15862        assert_eq!(observed_launch_nonce_source(&snapshot), None);
15863    }
15864    use super::{
15865        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15866        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15867        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15868    };
15869    use std::{ffi::OsStr, path::PathBuf};
15870    use tokio::process::Command;
15871
15872    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15873        ModuleSpec {
15874            module_id: "env-plan".to_string(),
15875            program: PathBuf::from("/nonexistent"),
15876            args: Vec::new(),
15877            env,
15878            reserved: false,
15879            reserved_prefixes: Vec::new(),
15880            protocol: ModuleProtocol::Subc,
15881            overlap: Default::default(),
15882        }
15883    }
15884
15885    #[test]
15886    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15887        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15888        let digest = LaunchSpecDigest::from_spec(&base);
15889        let mut reordered = base.clone();
15890        reordered.env.reverse();
15891        reordered.module_id = "different-id".into();
15892        reordered.reserved = true;
15893        reordered.overlap = Default::default();
15894        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15895        let mut delimiter_collision = base.clone();
15896        delimiter_collision.env[0] = ("A=B".into(), "C".into());
15897        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15898        let mut changed = base.clone();
15899        changed.protocol = ModuleProtocol::None;
15900        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15901        changed = base.clone();
15902        changed.args.push("--flag".into());
15903        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15904        changed = base.clone();
15905        changed.program = PathBuf::from("/other");
15906        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15907        changed = base.clone();
15908        changed.env[1].1 = "other".into();
15909        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15910        assert!(!format!("{digest:?}").contains("B=C"));
15911        assert!(!format!("{digest:?}").contains("[u8"));
15912    }
15913
15914    #[cfg(unix)]
15915    #[test]
15916    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15917        use std::os::unix::ffi::OsStringExt;
15918        let mut left = spec(Vec::new());
15919        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15920        let mut right = left.clone();
15921        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15922        assert_eq!(
15923            left.program.to_string_lossy(),
15924            right.program.to_string_lossy()
15925        );
15926        assert_ne!(
15927            LaunchSpecDigest::from_spec(&left),
15928            LaunchSpecDigest::from_spec(&right)
15929        );
15930    }
15931
15932    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
15933    /// one still gets its own.
15934    ///
15935    /// This is the narrow goal `env_clear()` was reached for, and the reason the
15936    /// fix is `env_remove` rather than deleting the line: an operator's ambient
15937    /// filter silently becoming an unconfigured module's log level is a real
15938    /// defect, just a much smaller one than clearing the environment.
15939    ///
15940    /// Asserted on the command plan rather than a spawned child because proving
15941    /// the ABSENCE of an inherited variable needs the parent's environment
15942    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
15943    /// removal as `(key, None)`, which is exactly the distinction wanted: not
15944    /// "absent because nobody set it" but "explicitly unset for the child".
15945    #[test]
15946    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15947        let mut command = Command::new("/nonexistent");
15948        apply_child_env(&mut command, &spec(Vec::new()));
15949        let removed = command
15950            .as_std()
15951            .get_envs()
15952            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15953        assert!(
15954            removed,
15955            "ambient CK_LOG must be explicitly removed for an unconfigured module"
15956        );
15957
15958        let mut configured = Command::new("/nonexistent");
15959        apply_child_env(
15960            &mut configured,
15961            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15962        );
15963        let effective = configured
15964            .as_std()
15965            .get_envs()
15966            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15967            .last()
15968            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15969        assert_eq!(
15970            effective,
15971            Some(Some("debug".to_string())),
15972            "a module's configured CK_LOG must survive the ambient removal"
15973        );
15974    }
15975
15976    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
15977    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
15978    /// the same reason as the CK_LOG test above.
15979    ///
15980    /// The argument is the load-bearing half: a stock binary exits on an
15981    /// unknown flag before it listens, so with `--subc` appended the mode
15982    /// could not supervise the one process it exists for. Found by the first
15983    /// conformance run (nats-server: `flag provided but not defined: -subc`).
15984    #[test]
15985    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15986        let connection_file = std::path::Path::new("/run/subc-connection.json");
15987        let handle = SupervisorHandle::new();
15988
15989        let mut none_spec = spec(Vec::new());
15990        none_spec.protocol = ModuleProtocol::None;
15991        let mut none = Command::new("/nonexistent");
15992        let none_handoff =
15993            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15994                .expect("protocol-none spawn args apply");
15995        assert!(
15996            none_handoff.is_none(),
15997            "protocol:none spawn must not receive a nonce descriptor"
15998        );
15999        #[cfg(windows)]
16000        assert!(!none.as_std().get_envs().any(|(key, value)| key
16001            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16002            && value.is_some()));
16003        assert!(
16004            !none.as_std().get_envs().any(|(key, value)| key
16005                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16006                && value.is_some()),
16007            "protocol:none spawn must not name a nonce descriptor"
16008        );
16009        let none_args: Vec<String> = none
16010            .as_std()
16011            .get_args()
16012            .map(|a| a.to_string_lossy().into_owned())
16013            .collect();
16014        assert!(
16015            !none_args.iter().any(|a| a == SUBC_ARG),
16016            "protocol:none argv must not carry --subc; got {none_args:?}"
16017        );
16018        let none_has_nonce = none
16019            .as_std()
16020            .get_envs()
16021            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16022        assert!(
16023            !none_has_nonce,
16024            "protocol:none spawn must not receive a launch nonce"
16025        );
16026        let none_has_module_id = none
16027            .as_std()
16028            .get_envs()
16029            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16030        assert!(
16031            none_has_module_id,
16032            "SUBC_MODULE_ID is inert and stays on every path"
16033        );
16034        assert!(
16035            handle.spawn_nonce(&none_spec.module_id).is_none(),
16036            "no nonce record for a process that will never present one"
16037        );
16038
16039        // Control: the subc-wire path is unchanged by the branch above.
16040        let wire_spec = spec(Vec::new());
16041        let mut wire = Command::new("/nonexistent");
16042        let wire_handoff =
16043            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16044                .expect("subc-wire spawn args apply");
16045        let wire_fd_env = wire
16046            .as_std()
16047            .get_envs()
16048            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16049            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16050        #[cfg(unix)]
16051        assert_eq!(
16052            wire_fd_env,
16053            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16054            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16055        );
16056        #[cfg(windows)]
16057        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16058        #[cfg(not(any(unix, windows)))]
16059        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16060        let wire_args: Vec<String> = wire
16061            .as_std()
16062            .get_args()
16063            .map(|a| a.to_string_lossy().into_owned())
16064            .collect();
16065        assert_eq!(
16066            wire_args,
16067            vec![
16068                SUBC_ARG.to_string(),
16069                connection_file.to_string_lossy().into_owned()
16070            ],
16071            "a subc-wire spawn still carries --subc <path>"
16072        );
16073        assert_eq!(
16074            wire.as_std()
16075                .get_envs()
16076                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16077            !cfg!(unix),
16078            "only Windows supplies the environment nonce"
16079        );
16080        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16081    }
16082
16083    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16084    /// spec tries to set it; only a swap candidate carries it.
16085    ///
16086    /// "Set it only on candidates" is not enough, because spawn applies the
16087    /// spec's env verbatim and the daemon's own environment is inherited: either
16088    /// could hand a plain restart the swap role, and a module reading it would
16089    /// warm on its long swap budget while callers wait. Asserted as an explicit
16090    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16091    /// test above gives.
16092    #[test]
16093    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16094        let role = |command: &Command| {
16095            command
16096                .as_std()
16097                .get_envs()
16098                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16099                .last()
16100                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16101        };
16102        let forged = spec(vec![(
16103            SUBC_SPAWN_ROLE_ENV.to_string(),
16104            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16105        )]);
16106
16107        let mut plain = Command::new("/nonexistent");
16108        apply_child_env(&mut plain, &forged);
16109        apply_spawn_role(&mut plain, SpawnRole::Plain);
16110        assert_eq!(
16111            role(&plain),
16112            Some(None),
16113            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16114        );
16115
16116        let mut candidate = Command::new("/nonexistent");
16117        apply_child_env(&mut candidate, &spec(Vec::new()));
16118        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16119        assert_eq!(
16120            role(&candidate),
16121            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16122        );
16123    }
16124
16125    /// Daemon-private capture retention keys never reach the child.
16126    ///
16127    /// cortexkit-log exposes retention as a Rust struct with no environment
16128    /// names, so these entries are supervisor metadata. Passing them through
16129    /// would invent a public child-process contract by accident.
16130    #[test]
16131    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16132        let mut command = Command::new("/nonexistent");
16133        apply_child_env(
16134            &mut command,
16135            &spec(vec![
16136                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16137                ("KEPT".to_string(), "yes".to_string()),
16138            ]),
16139        );
16140        let keys: Vec<String> = command
16141            .as_std()
16142            .get_envs()
16143            .filter(|(_, value)| value.is_some())
16144            .map(|(key, _)| key.to_string_lossy().into_owned())
16145            .collect();
16146        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16147        assert!(
16148            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16149            "daemon-private capture key leaked to the child: {keys:?}"
16150        );
16151    }
16152}
16153
16154#[cfg(test)]
16155mod jitter_tests {
16156    use super::jittered_health_delay;
16157    use std::{collections::HashSet, time::Duration};
16158
16159    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16160    /// that actually occur rather than invented ones.
16161    ///
16162    /// This is a SAMPLE, not a registry: the property under test is that distinct
16163    /// ids disperse, which holds for any set of distinct strings. Several entries
16164    /// are already historical (modules get renamed), and that costs nothing here --
16165    /// but it means a reader must not mistake this for the live module set, and a
16166    /// rename sweep will match it without there being anything to change.
16167    const FLEET: [&str; 14] = [
16168        "aft",
16169        "alfonso-core",
16170        "magic-context",
16171        "broca",
16172        "thalamus",
16173        "quota",
16174        "engram",
16175        "plexus",
16176        "cerebellum",
16177        "astrocyte",
16178        "synapse",
16179        "subc-mcp",
16180        "cortexkit-credentials",
16181        "subc-federation",
16182    ];
16183
16184    /// Probes must not converge after a fleet-wide restart.
16185    ///
16186    /// This is the property the jitter exists for: every module reconnects at
16187    /// once, and without dispersal all fourteen would then probe on the same
16188    /// tick forever. Nothing failed visibly when this went untested -- a
16189    /// convergent fleet still probes correctly, just in a burst, so the symptom
16190    /// is a periodic load spike that looks like whatever else is running.
16191    #[test]
16192    fn probe_delays_disperse_across_the_fleet() {
16193        let cadence = Duration::from_secs(30);
16194        let delays: HashSet<Duration> = FLEET
16195            .iter()
16196            .map(|id| jittered_health_delay(id, 0, cadence))
16197            .collect();
16198        assert_eq!(
16199            delays.len(),
16200            FLEET.len(),
16201            "every supervised module must land on its own probe offset"
16202        );
16203    }
16204
16205    /// The offset may only ever DELAY a probe, never bring it forward.
16206    ///
16207    /// A delay below the cadence would probe a module more often than
16208    /// configured, which is the opposite of what an operator asked for and
16209    /// would tighten the failure budget without anyone changing it.
16210    #[test]
16211    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16212        let cadence = Duration::from_secs(30);
16213        let span = cadence / 10;
16214        for id in FLEET {
16215            for probe_index in 0..8 {
16216                let delay = jittered_health_delay(id, probe_index, cadence);
16217                assert!(
16218                    delay >= cadence,
16219                    "{id}#{probe_index}: jitter must not shorten the cadence"
16220                );
16221                assert!(
16222                    delay < cadence + span,
16223                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16224                );
16225            }
16226        }
16227    }
16228
16229    /// A module keeps its offset across daemon restarts.
16230    ///
16231    /// The delay is derived rather than randomised precisely so a restart does
16232    /// not re-roll every module into a fresh chance of collision. A random
16233    /// source would satisfy the dispersal test above and quietly lose this.
16234    #[test]
16235    fn a_module_offset_is_stable_across_restarts() {
16236        let cadence = Duration::from_secs(30);
16237        for id in FLEET {
16238            assert_eq!(
16239                jittered_health_delay(id, 0, cadence),
16240                jittered_health_delay(id, 0, cadence),
16241                "{id}: the same module and probe index must produce the same offset"
16242            );
16243        }
16244    }
16245
16246    /// A zero cadence disables probing rather than producing a busy loop.
16247    #[test]
16248    fn zero_cadence_yields_zero_delay() {
16249        assert_eq!(
16250            jittered_health_delay("aft", 0, Duration::ZERO),
16251            Duration::ZERO
16252        );
16253    }
16254}
16255
16256#[cfg(all(test, target_os = "linux"))]
16257mod cgroup_placement_tests {
16258    use super::{
16259        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16260        SuperviseError, SupervisedChild,
16261    };
16262    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16263    use cortexkit_test_support::ScratchDir;
16264    use std::{
16265        fs, io,
16266        path::{Path, PathBuf},
16267        sync::{Arc, Mutex},
16268    };
16269    use tokio::process::Command;
16270
16271    #[tokio::test]
16272    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16273        use super::*;
16274        let dir = ScratchDir::new("unique-spawn-cgroups");
16275        let root = PathBuf::from(format!(
16276            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16277            std::process::id(),
16278            unix_ms_now()
16279        ));
16280        if let Err(error) = fs::create_dir(&root) {
16281            assert!(
16282                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16283                "required cgroup test cannot execute: {error}"
16284            );
16285            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16286            return;
16287        }
16288        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16289        let group_count = || {
16290            fs::read_dir(root.join("subc-modules"))
16291                .unwrap()
16292                .map(|entry| entry.unwrap().file_type().unwrap())
16293                .filter(|kind| kind.is_dir())
16294                .count()
16295        };
16296        let supervisor =
16297            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16298                .with_cgroup_placement(Some(placement.clone()));
16299        let runtime = supervisor.runtime_config();
16300        let mut spec = ModuleSpec {
16301            module_id: "unique-spawn".into(),
16302            program: PathBuf::from("/bin/sleep"),
16303            args: vec!["60".into()],
16304            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16305                .into_iter()
16306                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16307                .collect(),
16308            reserved: false,
16309            reserved_prefixes: vec![],
16310            protocol: ModuleProtocol::None,
16311            overlap: Default::default(),
16312        };
16313        let spawn = |spec: &ModuleSpec| {
16314            spawn_child(
16315                spec,
16316                None,
16317                None,
16318                &runtime.stderr_ring,
16319                None,
16320                &runtime.child_roster,
16321                Some(&placement),
16322            )
16323            .unwrap()
16324        };
16325        let mut live = spawn(&spec);
16326        for _ in 0..3 {
16327            // A new process can enter the old slot while retirement is pending.
16328            let next = spawn(&spec);
16329            assert_ne!(live.module_id, next.module_id);
16330            live.start_kill().unwrap();
16331            live.wait().await.unwrap();
16332            live = next;
16333            assert!(
16334                live.child.try_wait().unwrap().is_none(),
16335                "retiring the old slot must not kill the replacement"
16336            );
16337            assert_eq!(
16338                group_count(),
16339                1,
16340                "only the live spawn's cgroup should remain"
16341            );
16342        }
16343        supervisor.begin_daemon_shutdown();
16344        let reap = tokio::spawn(async move {
16345            live.wait().await.unwrap();
16346        });
16347        supervisor
16348            .end_children_for_daemon_shutdown(false, std::future::pending())
16349            .await;
16350        reap.await.unwrap();
16351        assert_eq!(group_count(), 0);
16352        // A normal exit uses the same tree-cleanup path as a killed spawn.
16353        spec.program = PathBuf::from("/bin/true");
16354        spec.args.clear();
16355        let fresh_roster = ChildRoster::default();
16356        let mut short = spawn_child(
16357            &spec,
16358            None,
16359            None,
16360            &runtime.stderr_ring,
16361            None,
16362            &fresh_roster,
16363            Some(&placement),
16364        )
16365        .unwrap();
16366        short.wait().await.unwrap();
16367        assert_eq!(group_count(), 0);
16368        spec.module_id = "_".repeat(255);
16369        let mut long_id = spawn_child(
16370            &spec,
16371            None,
16372            None,
16373            &runtime.stderr_ring,
16374            None,
16375            &fresh_roster,
16376            Some(&placement),
16377        )
16378        .unwrap();
16379        long_id.wait().await.unwrap();
16380        assert_eq!(
16381            group_count(),
16382            0,
16383            "valid long module IDs must not exceed cgroup NAME_MAX"
16384        );
16385        fs::remove_dir(root.join("subc-modules")).unwrap();
16386        fs::remove_dir(root).unwrap();
16387        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16388    }
16389
16390    #[test]
16391    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16392        let path = Path::new("/definitely-missing-subc-cgroup");
16393        let mut command = Command::new("true");
16394        let error = apply_cgroup_placement(
16395            &mut command,
16396            &ModuleSpec {
16397                module_id: "broken-cgroup".to_string(),
16398                program: PathBuf::from("true"),
16399                args: Vec::new(),
16400                env: Vec::new(),
16401                reserved: false,
16402                reserved_prefixes: Vec::new(),
16403                protocol: ModuleProtocol::Subc,
16404                overlap: Default::default(),
16405            },
16406            path,
16407        )
16408        .expect_err("a parent cgroup open failure must reject the supervised spawn");
16409        let reason = error.to_string();
16410
16411        assert!(
16412            matches!(error, SuperviseError::Cgroup { .. }),
16413            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16414        );
16415        assert!(
16416            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16417            "parent cgroup open failure must name cgroup.procs: {reason}"
16418        );
16419    }
16420
16421    #[tokio::test]
16422    async fn reaping_a_child_removes_its_empty_module_cgroup() {
16423        let root = ScratchDir::new("supervisor-reap-cgroup");
16424        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16425        let placement = subc_cgroup::prepare_at(&root)
16426            .expect("prepare scratch cgroup root")
16427            .expect("scratch root has a cgroup.procs marker");
16428        let module_id = "reaped-module";
16429        let module = placement
16430            .module_path(module_id)
16431            .expect("create scratch module cgroup");
16432        let child = Command::new("true")
16433            .env("XDG_DATA_HOME", root.path())
16434            .env("XDG_RUNTIME_DIR", root.path())
16435            .env("XDG_CONFIG_HOME", root.path())
16436            .spawn()
16437            .expect("spawn short-lived child");
16438        let pid = child.id().expect("spawned child has pid");
16439        let mut child = SupervisedChild {
16440            child,
16441            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16442            protocol: ModuleProtocol::Subc,
16443            module_id: module_id.to_string(),
16444            cgroup_placement: Some(placement),
16445            stdout_pump: None,
16446            stderr_pump: None,
16447            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16448            spawned_at: tokio::time::Instant::now(),
16449            registration_deadline: None,
16450            registration_watch: None,
16451            spawned_at_ms: 0,
16452            spawned_from: PathBuf::from("true"),
16453            launch_spec_digest: LaunchSpecDigest([0; 32]),
16454            spawned_file_identity: None,
16455            process_start_time: None,
16456            process_identity: None,
16457            pid,
16458            roster_guard: None,
16459            #[cfg(target_os = "macos")]
16460            privacy_exec: None,
16461            spawn_failure: None,
16462        };
16463
16464        child.wait().await.expect("reap short-lived child");
16465
16466        assert!(
16467            !module.exists(),
16468            "reaping the supervised child must remove its empty cgroup"
16469        );
16470    }
16471
16472    #[test]
16473    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16474        let root = ScratchDir::new("supervisor-non-empty-cgroup");
16475        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16476        let placement = subc_cgroup::prepare_at(&root)
16477            .expect("prepare scratch cgroup root")
16478            .expect("scratch root has a cgroup.procs marker");
16479        let module = placement
16480            .module_path("surviving-module")
16481            .expect("create scratch module cgroup");
16482        fs::write(module.join("surviving-process"), b"still present")
16483            .expect("make scratch cgroup non-empty");
16484        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16485
16486        remove_module_cgroup(&placement, "surviving-module");
16487
16488        let logs = crate::router::test_log::captured_logs(&logs);
16489        assert!(
16490            module.exists(),
16491            "failed removal must leave the cgroup intact"
16492        );
16493        assert!(
16494            logs.contains("could not remove module cgroup after process exit; continuing teardown")
16495                && logs.contains("surviving-module"),
16496            "best-effort removal must report the failure without returning it: {logs}"
16497        );
16498    }
16499
16500    #[test]
16501    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16502        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16503        let reason = SuperviseError::Spawn {
16504            program: PathBuf::from("/bin/true"),
16505            source: io::Error::from_raw_os_error(13),
16506            cgroup_path: Some(cgroup_path.clone()),
16507        }
16508        .to_string();
16509
16510        assert!(
16511            reason.contains(&cgroup_path.display().to_string()),
16512            "a pre_exec spawn failure must name the cgroup path: {reason}"
16513        );
16514    }
16515}
16516
16517#[cfg(test)]
16518mod spawn_subscriber_lag_tests {
16519    use super::*;
16520
16521    /// A subscriber whose connection stops draining is dropped once its frame
16522    /// channel fills. The client must learn that from a terminal Error frame
16523    /// after the frames already queued for it, not from a stream that simply
16524    /// goes quiet.
16525    #[tokio::test]
16526    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16527        let feed = SpawnEventFeed::default();
16528        feed.configure_incarnation("lag-incarnation".to_string());
16529        // A one-slot connection queue that nobody reads until the emits are
16530        // done: the forwarder parks on it and the subscriber channel fills.
16531        let (tx, mut rx) = mpsc::channel(1);
16532        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16533            .expect("subscribe");
16534        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16535        for index in 0..emitted {
16536            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16537            // Let the forwarder take what it can so the fill point is the
16538            // subscriber channel, not a scheduling accident.
16539            tokio::task::yield_now().await;
16540        }
16541        assert_eq!(
16542            feed.subscriber_count(),
16543            0,
16544            "the lagged subscriber must be removed"
16545        );
16546
16547        let mut data = Vec::new();
16548        let mut last = None;
16549        loop {
16550            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16551                .await
16552                .expect("the forwarder must finish once the subscriber is dropped");
16553            let Some(outbound) = next else { break };
16554            let frame = outbound.frame;
16555            if frame.header.ty == FrameType::StreamData {
16556                assert!(last.is_none(), "no data may follow the terminal frame");
16557                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16558                data.push(event.cursor.seq);
16559            } else {
16560                assert!(last.is_none(), "exactly one terminal frame");
16561                last = Some(frame);
16562            }
16563        }
16564        assert!(!data.is_empty(), "queued frames drain before the terminal");
16565        for pair in data.windows(2) {
16566            assert_eq!(
16567                pair[1],
16568                pair[0] + 1,
16569                "queued frames arrive dense and in order"
16570            );
16571        }
16572        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16573        assert_eq!(terminal.header.ty, FrameType::Error);
16574        assert_eq!(terminal.header.corr, 7);
16575        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16576        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16577        let detail = body.detail.expect("lagged error carries detail");
16578        assert_eq!(
16579            detail["first_undelivered_cursor"]["seq"],
16580            data.last().unwrap() + 1,
16581            "the named cursor is the first event the subscriber did not receive"
16582        );
16583        assert_eq!(
16584            detail["first_undelivered_cursor"]["daemon_incarnation"],
16585            "lag-incarnation"
16586        );
16587    }
16588}
16589
16590#[cfg(test)]
16591mod terminal_history_read_concurrency_tests {
16592    use super::*;
16593    use crate::terminal_journal::read_pause;
16594    use cortexkit_test_support::ScratchDir;
16595    use std::sync::mpsc as std_mpsc;
16596
16597    fn journaled_ring(
16598        journal: &Arc<crate::terminal_journal::TerminalJournal>,
16599    ) -> Arc<Mutex<TerminalRing>> {
16600        Arc::new(Mutex::new(
16601            TerminalRing::new(TerminalRingConfig::default(), 1)
16602                .with_journal(Some(Arc::clone(journal))),
16603        ))
16604    }
16605
16606    fn crash(at_ms: u64) -> ExitReport {
16607        ExitReport {
16608            kind: ExitKind::Crash,
16609            code: Some(1),
16610            signal: None,
16611            at_ms,
16612        }
16613    }
16614
16615    /// Record an exit on another thread and report whether it finished within
16616    /// `bound`. The recorder thread is left running if it did not.
16617    fn record_within(
16618        module_id: &'static str,
16619        ring: &Arc<Mutex<TerminalRing>>,
16620        at_ms: u64,
16621        bound: Duration,
16622    ) -> bool {
16623        let ring = Arc::clone(ring);
16624        let (done, done_rx) = std_mpsc::channel();
16625        std::thread::spawn(move || {
16626            record_terminal(
16627                module_id,
16628                &ring,
16629                &SpawnEventFeed::default(),
16630                &crash(at_ms),
16631                TerminalDisposition::Restarting,
16632            );
16633            let _ = done.send(());
16634        });
16635        done_rx.recv_timeout(bound).is_ok()
16636    }
16637
16638    /// A history read in progress must not hold the journal writer (which every
16639    /// module's exit recording needs) or the module's own ring. Exits recorded
16640    /// while the read is paused complete promptly; the paused read answers as of
16641    /// the moment it started, and the next read has each exit exactly once.
16642    #[test]
16643    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16644        let dir = ScratchDir::new("terminal-history-concurrent-read");
16645        let path = dir.join("terminals.jsonl");
16646        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16647            path.clone(),
16648            "daemon".into(),
16649        ));
16650        let reader_ring = journaled_ring(&journal);
16651        let other_ring = journaled_ring(&journal);
16652        assert!(record_within(
16653            "reader-module",
16654            &reader_ring,
16655            10,
16656            Duration::from_secs(5)
16657        ));
16658
16659        let (started, release) = read_pause::install(&path);
16660        let reading = {
16661            let ring = Arc::clone(&reader_ring);
16662            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16663        };
16664        started
16665            .recv_timeout(Duration::from_secs(5))
16666            .expect("the history read reached its pause");
16667
16668        let bound = Duration::from_secs(1);
16669        assert!(
16670            record_within("other-module", &other_ring, 20, bound),
16671            "another module's exit waited on a history read (journal writer held)"
16672        );
16673        assert!(
16674            record_within("reader-module", &reader_ring, 30, bound),
16675            "the read module's own exit waited on its history read (ring held)"
16676        );
16677
16678        drop(release);
16679        let paused = reading.join().unwrap();
16680        assert_eq!(
16681            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16682            vec![10],
16683            "an exit recorded after the read began lands in neither half of it"
16684        );
16685        assert_eq!(paused.journal_skipped_lines, 0);
16686        assert_eq!(paused.journal_read_errors, 0);
16687
16688        let after = durable_terminal_history_of(&reader_ring, "reader-module");
16689        assert_eq!(
16690            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16691            vec![10, 30],
16692            "the next read merges ring and journal with no duplicate"
16693        );
16694        assert_eq!(after.journal_skipped_lines, 0);
16695    }
16696}
16697
16698/// What a restart does with the exited process's output readers. These drive
16699/// the same settle functions the supervisor calls, with a reader the test
16700/// holds, so a reader that has not been scheduled by the bound is a controlled
16701/// input rather than something only a loaded machine produces.
16702#[cfg(test)]
16703mod stderr_settle_tests {
16704    use std::{
16705        future::Future,
16706        io,
16707        pin::Pin,
16708        sync::{Arc, Mutex},
16709        task::{Context, Poll},
16710        time::Duration,
16711    };
16712
16713    use tokio::{
16714        io::{AsyncRead, ReadBuf},
16715        sync::oneshot,
16716        time::Instant,
16717    };
16718
16719    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16720    use crate::stderr_tail::{
16721        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16722        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16723    };
16724
16725    const BOUND: Duration = Duration::from_millis(250);
16726
16727    /// Yields `before`, then stays pending until the gate is released, then
16728    /// yields `after` and reaches EOF. The bytes after the gate were written
16729    /// by a process that has already exited; only the reader is behind.
16730    struct HeldReader {
16731        before: Option<Vec<u8>>,
16732        gate: Option<oneshot::Receiver<()>>,
16733        after: io::Cursor<Vec<u8>>,
16734    }
16735
16736    impl AsyncRead for HeldReader {
16737        fn poll_read(
16738            mut self: Pin<&mut Self>,
16739            cx: &mut Context<'_>,
16740            buf: &mut ReadBuf<'_>,
16741        ) -> Poll<io::Result<()>> {
16742            if let Some(bytes) = self.before.take() {
16743                buf.put_slice(&bytes);
16744                return Poll::Ready(Ok(()));
16745            }
16746            if let Some(gate) = self.gate.as_mut() {
16747                match Pin::new(gate).poll(cx) {
16748                    Poll::Pending => return Poll::Pending,
16749                    Poll::Ready(_) => self.gate = None,
16750                }
16751            }
16752            Pin::new(&mut self.after).poll_read(cx, buf)
16753        }
16754    }
16755
16756    struct DiscardSink;
16757
16758    impl OutputSink for DiscardSink {
16759        fn write_line(&mut self, _line: &[u8]) {}
16760    }
16761
16762    fn line(text: &str) -> TailEntry {
16763        TailEntry::Line {
16764            text: text.to_string(),
16765            truncated: false,
16766            at_ms: None,
16767        }
16768    }
16769
16770    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16771        ring.lock().unwrap()
16772    }
16773
16774    /// Start a reader for a new process generation that delivers `before`
16775    /// immediately and `after` only once the returned sender fires (or is
16776    /// dropped).
16777    fn held_pump(
16778        ring: &Arc<Mutex<StderrRing>>,
16779        before: &str,
16780        after: &str,
16781    ) -> (StderrPump, oneshot::Sender<()>) {
16782        let generation = lock(ring).begin_process();
16783        let (release, gate) = oneshot::channel();
16784        let reader = HeldReader {
16785            before: Some(before.as_bytes().to_vec()),
16786            gate: Some(gate),
16787            after: io::Cursor::new(after.as_bytes().to_vec()),
16788        };
16789        let task = tokio::spawn(pump_stderr_to(
16790            reader,
16791            Arc::clone(ring),
16792            generation,
16793            DiscardSink,
16794        ));
16795        (StderrPump { task, generation }, release)
16796    }
16797
16798    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16799        for _ in 0..1000 {
16800            if done(&lock(ring)) {
16801                return;
16802            }
16803            tokio::time::sleep(Duration::from_millis(1)).await;
16804        }
16805        panic!(
16806            "ring never reached the expected state: {:?}",
16807            lock(ring).snapshot(None, None)
16808        );
16809    }
16810
16811    #[tokio::test(start_paused = true)]
16812    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16813        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16814        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16815
16816        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16817        let before_release = lock(&ring).snapshot(None, None);
16818        assert!(
16819            matches!(before_release.capture, CaptureState::Incomplete { .. }),
16820            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16821        );
16822
16823        // The restart: the next process starts and writes before the old
16824        // reader catches up.
16825        let next = lock(&ring).begin_process();
16826        lock(&ring).push_line_from(next, "next process booting");
16827        release.send(()).unwrap();
16828        wait_until(&ring, |ring| {
16829            ring.snapshot(None, None).capture == CaptureState::Captured
16830        })
16831        .await;
16832
16833        assert_eq!(
16834            untimed(lock(&ring).snapshot(None, None).entries),
16835            vec![
16836                line("booting"),
16837                line("config error: missing storage"),
16838                TailEntry::ProcessStart,
16839                line("next process booting"),
16840            ],
16841            "the crash's last line must survive a slow reader and stay in the crashed process's section"
16842        );
16843    }
16844
16845    #[tokio::test(start_paused = true)]
16846    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16847    ) {
16848        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16849        // `_held` is never fired: a descendant keeps the pipe open for the
16850        // whole test.
16851        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16852
16853        let started = Instant::now();
16854        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16855        assert_eq!(
16856            started.elapsed(),
16857            BOUND,
16858            "the restart must wait exactly the bound for a pipe that stays open, no longer"
16859        );
16860
16861        let next = lock(&ring).begin_process();
16862        lock(&ring).push_line_from(next, "next process booting");
16863        tokio::time::sleep(Duration::from_secs(60)).await;
16864
16865        let snapshot = lock(&ring).snapshot(None, None);
16866        match &snapshot.capture {
16867            CaptureState::Incomplete { reason } => assert!(
16868                reason.contains("had not reached EOF") && reason.contains("250ms"),
16869                "the reason must say what is missing and after how long: {reason}"
16870            ),
16871            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16872        }
16873        assert_eq!(
16874            untimed(snapshot.entries),
16875            vec![
16876                line("parent exiting"),
16877                TailEntry::ProcessStart,
16878                line("next process booting"),
16879            ]
16880        );
16881    }
16882
16883    #[tokio::test(start_paused = true)]
16884    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16885        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16886        let (pump, release) = held_pump(&ring, "one\n", "two\n");
16887        release.send(()).unwrap();
16888
16889        settle_stderr_pump("clean", &ring, pump, BOUND).await;
16890
16891        let snapshot = lock(&ring).snapshot(None, None);
16892        assert_eq!(snapshot.capture, CaptureState::Captured);
16893        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16894    }
16895
16896    #[tokio::test(start_paused = true)]
16897    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16898        const BURST: usize = 150;
16899        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16900        let path = capture.join("two-pipe.stderr.log");
16901        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16902        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16903        let generation = lock(&ring).begin_process();
16904        let stdout = (0..BURST)
16905            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16906            .collect::<String>();
16907        let stderr = (0..BURST)
16908            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16909            .collect::<String>();
16910        // Model a child that has written both bursts and exited. Hold the stdout
16911        // reader after one line and a partial prefix, as if pipe readiness or
16912        // capture-file writes had delayed it. The final line also needs EOF to
16913        // flush: neither complete nor partial buffered output may be discarded.
16914        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16915        let (release, gate) = oneshot::channel();
16916        let reader = HeldReader {
16917            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16918            gate: Some(gate),
16919            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16920        };
16921        let (finished, completion) = oneshot::channel();
16922        let stdout_sink = sink.clone();
16923        let pump = tokio::spawn(async move {
16924            pump_stdout_to(reader, stdout_sink).await;
16925            finished.send(()).unwrap();
16926        });
16927        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16928
16929        let started = Instant::now();
16930        settle_stdout_pump("two-pipe", pump, BOUND).await;
16931        assert_eq!(
16932            started.elapsed(),
16933            BOUND,
16934            "a late reader must not hold up restart"
16935        );
16936        assert_eq!(
16937            std::fs::read_to_string(&path).unwrap().lines().count(),
16938            BURST + 1
16939        );
16940
16941        release
16942            .send(())
16943            .expect("stdout reader must survive the exit drain bound");
16944        tokio::time::timeout(Duration::from_secs(1), completion)
16945            .await
16946            .expect("released stdout reader must reach EOF")
16947            .expect("stdout capture must finish at EOF, not be cancelled");
16948        let contents = std::fs::read_to_string(&path).unwrap();
16949        let mut lines = contents
16950            .lines()
16951            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16952            .collect::<Vec<_>>();
16953        let mut expected = (0..BURST)
16954            .flat_map(|i| {
16955                [
16956                    format!("out-{i:04}-{}", "o".repeat(64)),
16957                    format!("err-{i:04}-{}", "e".repeat(64)),
16958                ]
16959            })
16960            .collect::<Vec<_>>();
16961        lines.sort_unstable();
16962        expected.sort_unstable();
16963        assert_eq!(
16964            lines, expected,
16965            "both pipes must retain every complete line exactly once"
16966        );
16967    }
16968}
16969
16970/// Containment of a module's process tree (issue #109).
16971///
16972/// The behaviour these defend against is a module helper surviving its module:
16973/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
16974/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
16975/// compounds it.
16976///
16977/// They run against the SUPERVISOR rather than the job-object crate because the
16978/// claim is about teardown: a crate-level test proves a job can reap a tree, not
16979/// that the daemon's drain path reaches it.
16980///
16981/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
16982/// lane there is a separate containment path with its own tests.
16983#[cfg(all(test, windows))]
16984mod job_containment_tests {
16985    use super::*;
16986    use cortexkit_test_support::ScratchDir;
16987    use std::{
16988        path::{Path, PathBuf},
16989        sync::{Arc, Mutex},
16990        time::{Duration, Instant},
16991    };
16992
16993    /// The stub, expected beside this test executable.
16994    ///
16995    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
16996    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
16997    /// failure then reads as a broken test rather than an unbuilt dependency.
16998    fn stub_path() -> PathBuf {
16999        let mut path = std::env::current_exe().expect("current_exe available in tests");
17000        path.pop();
17001        path.pop();
17002        path.push("fake-aft-stub.exe");
17003        assert!(
17004            path.exists(),
17005            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17006             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17007            path.display()
17008        );
17009        path
17010    }
17011
17012    /// Poll for the grandchild pid the stub records, and parse it.
17013    fn read_grandchild_pid(path: &Path) -> u32 {
17014        let deadline = Instant::now() + Duration::from_secs(10);
17015        loop {
17016            if let Ok(contents) = std::fs::read_to_string(path) {
17017                if let Ok(pid) = contents.trim().parse() {
17018                    return pid;
17019                }
17020            }
17021            assert!(
17022                Instant::now() < deadline,
17023                "the stub never recorded a grandchild pid at {}",
17024                path.display()
17025            );
17026            std::thread::sleep(Duration::from_millis(10));
17027        }
17028    }
17029
17030    /// Everything one fixture run needs, so the two tests below differ in exactly
17031    /// one place: whether the child is contained.
17032    struct Fixture {
17033        _dir: ScratchDir,
17034        module_id: String,
17035        grandchild: u32,
17036        child: Option<SupervisedChild>,
17037        registry: Arc<Registry>,
17038        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17039        terminal_ring: Arc<Mutex<TerminalRing>>,
17040        spawn_events: SpawnEventFeed,
17041    }
17042
17043    fn fixture(label: &str, module_id: &str) -> Fixture {
17044        let dir = ScratchDir::new(label);
17045        let pid_file = dir.join("grandchild.pid");
17046        let supervisor = Supervisor::new_for_test(
17047            Arc::new(Registry::default()),
17048            RestartPolicy::new(3, Duration::ZERO),
17049        );
17050        let runtime = supervisor.runtime_config();
17051        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17052        let spec = ModuleSpec {
17053            module_id: module_id.to_string(),
17054            program: stub_path(),
17055            // Zero args deliberately: a `--subc` argument would make the stub dial
17056            // a daemon that is not there, and the failure would land in the same
17057            // stderr ring this fixture exists to keep quiet.
17058            args: Vec::new(),
17059            env: vec![
17060                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17061                (
17062                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17063                    pid_file.display().to_string(),
17064                ),
17065            ],
17066            reserved: false,
17067            reserved_prefixes: Vec::new(),
17068            protocol: ModuleProtocol::Subc,
17069            overlap: Default::default(),
17070        };
17071        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17072            .expect("spawn the supervised fixture");
17073        let grandchild = read_grandchild_pid(&pid_file);
17074        Fixture {
17075            _dir: dir,
17076            module_id: module_id.to_string(),
17077            grandchild,
17078            child: Some(child),
17079            registry: Arc::new(Registry::default()),
17080            snapshot,
17081            terminal_ring: Arc::clone(&runtime.terminal_ring),
17082            spawn_events: SpawnEventFeed::default(),
17083        }
17084    }
17085
17086    impl Fixture {
17087        /// Drain through the supervisor's own teardown path.
17088        async fn drain(&mut self) {
17089            let child = self
17090                .child
17091                .take()
17092                .expect("the fixture child is still present");
17093            drain_child_to_state(
17094                &self.module_id,
17095                ModuleProtocol::Subc,
17096                // No forwarding table in this fixture, so nothing reaches the
17097                // child over a connection.
17098                StopNotice::NotSent,
17099                &self.registry,
17100                None,
17101                &self.snapshot,
17102                &self.terminal_ring,
17103                &self.spawn_events,
17104                child,
17105                Duration::from_millis(500),
17106                ModuleState::Stopped,
17107                Some(false),
17108                &FailureContext::operator(),
17109            )
17110            .await
17111            .expect("drain the supervised fixture");
17112        }
17113    }
17114
17115    /// Teardown reaps the grandchild, not merely the direct child.
17116    ///
17117    /// This is the assertion the change exists for. Before containment the
17118    /// grandchild survived: it is a separate process, and `start_kill` is
17119    /// `TerminateProcess` scoped to one pid.
17120    #[tokio::test]
17121    async fn teardown_reaps_the_grandchild() {
17122        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17123        let grandchild = fixture.grandchild;
17124
17125        assert!(
17126            subc_jobobject::process_exists(grandchild),
17127            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17128        );
17129
17130        fixture.drain().await;
17131
17132        assert!(
17133            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17134            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17135        );
17136    }
17137
17138    /// The mutation control: with containment withheld, the grandchild survives
17139    /// the same kill.
17140    ///
17141    /// This is the defect reproduction from #109 — a direct-child kill reaches
17142    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17143    /// supervisor because `spawn_and_mark_running` now always contains on
17144    /// Windows, which is the point: there is no longer a path that spawns
17145    /// uncontained, so the control has to construct one.
17146    ///
17147    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17148    /// grandchild ever dies here, that test is passing for a reason unrelated to
17149    /// the job object and the containment claim is unproven.
17150    #[test]
17151    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17152        let dir = ScratchDir::new("teardown-uncontained");
17153        let pid_file = dir.join("grandchild.pid");
17154        let mut child = std::process::Command::new(stub_path())
17155            .env("FAKE_AFT_NEVER_CONNECT", "1")
17156            .env(
17157                "FAKE_AFT_GRANDCHILD_PID_FILE",
17158                pid_file.display().to_string(),
17159            )
17160            .stdin(std::process::Stdio::null())
17161            .stdout(std::process::Stdio::null())
17162            .stderr(std::process::Stdio::null())
17163            .spawn()
17164            .expect("spawn the uncontained fixture");
17165        let grandchild = read_grandchild_pid(&pid_file);
17166
17167        // Exactly what the pre-fix teardown did: kill the direct child.
17168        child.kill().expect("kill the direct child");
17169        let _ = child.wait();
17170
17171        assert!(
17172            subc_jobobject::process_exists(grandchild),
17173            "grandchild {grandchild} died with the direct child, so this control no longer \
17174             distinguishes contained from uncontained teardown and the regression test is \
17175             passing vacuously"
17176        );
17177
17178        // The orphan this control demonstrates is the leak the fix prevents, so
17179        // the control must not leave one behind.
17180        kill_tree(grandchild);
17181    }
17182
17183    /// Crash durability: closing the containment handle reaps the tree with no
17184    /// teardown code running at all.
17185    ///
17186    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17187    /// call anything — and it is why containment is a kernel property of the
17188    /// handle rather than a step in the drain. Discovered by getting the
17189    /// mutation control wrong: clearing `job` to "disable" containment instead
17190    /// killed the tree, which is the guarantee, not a mistake.
17191    #[tokio::test]
17192    async fn dropping_containment_reaps_the_grandchild() {
17193        let mut fixture = fixture("drop-containment", "tree-drop");
17194        let grandchild = fixture.grandchild;
17195
17196        assert!(subc_jobobject::process_exists(grandchild));
17197
17198        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17199        fixture.child.as_mut().expect("child present").job = None;
17200
17201        assert!(
17202            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17203            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17204             crash would leave the tree behind"
17205        );
17206    }
17207
17208    /// Kill a pid and its tree, then confirm it is gone.
17209    fn kill_tree(pid: u32) {
17210        let _ = std::process::Command::new("taskkill.exe")
17211            .args(["/PID", &pid.to_string(), "/T", "/F"])
17212            .stdin(std::process::Stdio::null())
17213            .stdout(std::process::Stdio::null())
17214            .stderr(std::process::Stdio::null())
17215            .status();
17216        assert!(
17217            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17218            "could not clean up grandchild {pid}"
17219        );
17220    }
17221}
17222
17223#[cfg(test)]
17224mod privacy_trampoline_configuration_tests {
17225    #[tokio::test]
17226    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17227    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17228        #[cfg(target_os = "macos")]
17229        {
17230            let supervisor = super::Supervisor::new(
17231                std::sync::Arc::new(crate::Registry::default()),
17232                super::RestartPolicy::default(),
17233            );
17234            let error = supervisor.spawn(spec()).unwrap_err();
17235            assert!(
17236                error
17237                    .to_string()
17238                    .contains("no privacy trampoline configured"),
17239                "{error}"
17240            );
17241        }
17242    }
17243
17244    #[tokio::test]
17245    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17246    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17247        #[cfg(target_os = "macos")]
17248        {
17249            let supervisor = super::Supervisor::new(
17250                std::sync::Arc::new(crate::Registry::default()),
17251                super::RestartPolicy::default(),
17252            )
17253            .with_privacy_trampoline(std::env::current_exe().unwrap());
17254            let error = supervisor.spawn(spec()).unwrap_err();
17255            assert!(
17256                error
17257                    .to_string()
17258                    .contains("binary does not implement the privacy trampoline protocol"),
17259                "{error}"
17260            );
17261        }
17262    }
17263
17264    #[cfg(target_os = "macos")]
17265    fn spec() -> super::ModuleSpec {
17266        super::ModuleSpec {
17267            module_id: "privacy-configuration".into(),
17268            program: "/bin/sleep".into(),
17269            args: vec!["30".into()],
17270            env: vec![],
17271            reserved: false,
17272            reserved_prefixes: vec![],
17273            protocol: subc_control::ModuleProtocol::None,
17274            overlap: super::ModuleOverlap::Exclusive,
17275        }
17276    }
17277}
17278
17279#[cfg(test)]
17280mod privacy_exec_boundary_tests {
17281    #[cfg(target_os = "macos")]
17282    use super::*;
17283    #[cfg(target_os = "macos")]
17284    use std::{
17285        io::{Read, Write},
17286        net::{TcpListener, TcpStream},
17287    };
17288
17289    /// Unit-test-only pause at the actual early image read, not at a later
17290    /// status read. Production supervisors never inspect this environment key.
17291    #[cfg(target_os = "macos")]
17292    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17293        if let Some((_, path)) = spec
17294            .env
17295            .iter()
17296            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17297        {
17298            let mut barrier = TcpStream::connect(path).unwrap();
17299            barrier
17300                .set_read_timeout(Some(Duration::from_secs(30)))
17301                .unwrap();
17302            barrier.write_all(&pid.to_ne_bytes()).unwrap();
17303            let mut release = [0];
17304            barrier.read_exact(&mut release).unwrap();
17305            assert_eq!(&release, b"X");
17306        }
17307    }
17308
17309    #[cfg(target_os = "macos")]
17310    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17311        ModuleSpec {
17312            module_id: "privacy-boundary".into(),
17313            program: program.into(),
17314            args: args.iter().map(|arg| (*arg).into()).collect(),
17315            env: vec![],
17316            reserved: false,
17317            reserved_prefixes: vec![],
17318            protocol: ModuleProtocol::None,
17319            overlap: ModuleOverlap::Exclusive,
17320        }
17321    }
17322
17323    #[cfg(target_os = "macos")]
17324    async fn accept(listener: TcpListener) -> TcpStream {
17325        // Socket readiness, not elapsed time, establishes both pause points.
17326        let listener = tokio::net::TcpListener::from_std({
17327            listener.set_nonblocking(true).unwrap();
17328            listener
17329        })
17330        .unwrap();
17331        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17332            .await
17333            .unwrap()
17334            .unwrap();
17335        let stream = stream.into_std().unwrap();
17336        stream.set_nonblocking(false).unwrap();
17337        stream
17338            .set_read_timeout(Some(Duration::from_secs(30)))
17339            .unwrap();
17340        stream
17341    }
17342
17343    #[tokio::test]
17344    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17345    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17346        #[cfg(target_os = "macos")]
17347        {
17348            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17349            // Loopback sockets also work when the replay adapter's TMPDIR is
17350            // longer than Darwin's Unix-domain socket path limit.
17351            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17352            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17353            let record = root.join("live-children.json");
17354            let supervisor =
17355                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17356                    .with_live_children_record(&record);
17357            let runtime = supervisor.runtime_config();
17358            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17359            let mut spec = spec("/bin/sleep", &["30"]);
17360            spec.env = vec![
17361                (
17362                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17363                    exec_listener.local_addr().unwrap().to_string(),
17364                ),
17365                (
17366                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17367                    sample_listener.local_addr().unwrap().to_string(),
17368                ),
17369            ];
17370            let spawn = tokio::task::spawn_blocking(move || {
17371                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17372            });
17373            let mut sample = accept(sample_listener).await;
17374            let mut pid = [0; 4];
17375            sample.read_exact(&mut pid).unwrap();
17376            let pid = u32::from_ne_bytes(pid);
17377            let mut exec = accept(exec_listener).await;
17378            let mut ready = [0];
17379            exec.read_exact(&mut ready).unwrap();
17380            assert_eq!(&ready, b"R");
17381            // The early read is guaranteed to see a real, non-null trampoline
17382            // image: the fixture has reached its barrier and cannot exec yet.
17383            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17384            assert_eq!(
17385                observe_spawned_image(pid).unwrap().executable,
17386                Some(trampoline)
17387            );
17388            sample.write_all(b"X").unwrap();
17389            let mut child = spawn.await.unwrap();
17390            let early = crate::live_children::read_record(&record).unwrap();
17391            assert_eq!(early.len(), 1);
17392            assert_eq!(early[0].pid, pid);
17393            assert_eq!(
17394                early[0].executable, None,
17395                "unconfirmed trampoline image entered the roster"
17396            );
17397            assert!(child.report_ready.get().is_none());
17398            // The barrier's duration is unrelated to the production five-second
17399            // exec budget. Start the test's confirmation budget upon release.
17400            child.privacy_exec.as_mut().unwrap().deadline =
17401                tokio::time::Instant::now() + Duration::from_secs(30);
17402            exec.write_all(b"X").unwrap();
17403            child.confirm_privacy_exec().await;
17404            assert_eq!(child.spawn_failure, None);
17405            assert!(child.report_ready.get().is_some());
17406            let confirmed = crate::live_children::read_record(&record).unwrap();
17407            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17408            assert_ne!(module, trampoline);
17409            assert_eq!(confirmed[0].executable, Some(module.into()));
17410            child.start_kill().unwrap();
17411            child.wait().await.unwrap();
17412            child.release_roster();
17413        }
17414    }
17415
17416    #[tokio::test]
17417    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17418    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17419        #[cfg(target_os = "macos")]
17420        {
17421            let registry = Arc::new(Registry::default());
17422            let policy = RestartPolicy::new(0, Duration::ZERO);
17423            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
17424            let runtime = supervisor.runtime_config();
17425            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17426            let spec = spec("/bin/sh", &["-c", "exit 121"]);
17427            // Drive spawn and confirmation separately instead of starting the
17428            // monitor. WNOWAIT observes a real exit without consuming its status,
17429            // so confirmation's first try_wait must take the already-exited arm.
17430            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17431            let pid = child.pid;
17432            tokio::task::spawn_blocking(move || {
17433                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17434            })
17435            .await
17436            .unwrap()
17437            .unwrap();
17438            child.privacy_exec.as_mut().unwrap().deadline =
17439                tokio::time::Instant::now() + Duration::from_secs(30);
17440            let status = child.wait().await.unwrap();
17441            assert_eq!(status.code(), Some(121));
17442            assert!(child.privacy_exec.is_none());
17443            assert!(
17444                child.report_ready.get().is_none(),
17445                "an exited module must not publish a live pid"
17446            );
17447            let report = classify_reaped_child_exit(&snapshot, &child, &status);
17448            on_child_exit(
17449                &spec,
17450                policy,
17451                &registry,
17452                &snapshot,
17453                &runtime.terminal_ring,
17454                &runtime.spawn_events,
17455                &runtime.child_roster,
17456                report,
17457                true,
17458            )
17459            .await;
17460            let state = lock_snapshot(&snapshot).unwrap();
17461            assert_eq!(state.state, ModuleState::Failed);
17462            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17463            assert_eq!(state.reported_pid(), None);
17464            drop(state);
17465            let history = runtime.terminal_ring.lock().unwrap().snapshot();
17466            assert_eq!(history.entries.len(), 1);
17467            let terminal = &history.entries[0];
17468            assert_eq!(terminal.exit_code, Some(121));
17469            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17470            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17471            assert_eq!(
17472                terminal.disposition_detail.as_deref(),
17473                Some(policy.budget_exhausted_detail().as_str()),
17474                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17475            );
17476            assert_eq!(child.spawn_failure, None);
17477            child.release_roster();
17478        }
17479    }
17480}
17481
17482/// The daemon's real spawn path hands a subc-wire child its launch nonce on
17483/// descriptor 3, without an environment copy. The shell records the nonce
17484/// and its environment after exec so these tests observe the real handover.
17485#[cfg(all(test, unix))]
17486mod launch_nonce_descriptor_tests {
17487    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17488    use crate::stderr_tail::{StderrRing, StderrTailConfig};
17489    use cortexkit_test_support::ScratchDir;
17490    use std::{
17491        path::PathBuf,
17492        sync::{Arc, Mutex},
17493        time::{Duration, Instant},
17494    };
17495
17496    async fn probe(role: super::SpawnRole) {
17497        let scratch = ScratchDir::new("launch-nonce-descriptor");
17498        let fd_copy = scratch.join("from-descriptor");
17499        let env_copy = scratch.join("environment");
17500        let script = format!(
17501            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17502            fd = fd_copy.display(), env = env_copy.display(),
17503        );
17504        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17505        let spec = ModuleSpec {
17506            module_id: "nonce-descriptor-probe".to_string(),
17507            program: PathBuf::from("/bin/sh"),
17508            args: vec!["-c".to_string(), script],
17509            env: vec![
17510                xdg("XDG_DATA_HOME"),
17511                xdg("XDG_RUNTIME_DIR"),
17512                xdg("XDG_CONFIG_HOME"),
17513                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17514            ],
17515            reserved: true,
17516            reserved_prefixes: Vec::new(),
17517            protocol: ModuleProtocol::Subc,
17518            overlap: Default::default(),
17519        };
17520        let handle = SupervisorHandle::new();
17521        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17522        let roster = ChildRoster::default();
17523        #[cfg(target_os = "macos")]
17524        {
17525            let path = super::test_privacy_trampoline();
17526            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17527        }
17528        let child = super::spawn_child_in_slot(
17529            &spec,
17530            None,
17531            Some(&handle),
17532            &ring,
17533            None,
17534            &roster,
17535            #[cfg(target_os = "linux")]
17536            None,
17537            role,
17538            matches!(role, super::SpawnRole::SwapCandidate),
17539        )
17540        .expect("spawn probe");
17541        let deadline = Instant::now() + Duration::from_secs(10);
17542        while !(fd_copy.exists() && env_copy.exists()) {
17543            assert!(Instant::now() < deadline, "probe never wrote its copies");
17544            tokio::time::sleep(Duration::from_millis(20)).await;
17545        }
17546        let nonce = std::fs::read_to_string(fd_copy).unwrap();
17547        assert!(!nonce.is_empty());
17548        let environment = std::fs::read_to_string(env_copy).unwrap();
17549        assert!(environment
17550            .lines()
17551            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17552        let copy = environment
17553            .lines()
17554            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17555        assert_eq!(
17556            copy, None,
17557            "Unix children must never receive the environment nonce"
17558        );
17559        if matches!(role, super::SpawnRole::Plain) {
17560            assert_eq!(
17561                handle.spawn_nonce(&spec.module_id).as_deref(),
17562                Some(nonce.as_str())
17563            );
17564        }
17565        drop(child);
17566    }
17567
17568    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17569    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17570        probe(super::SpawnRole::Plain).await;
17571    }
17572
17573    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17574    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17575        probe(super::SpawnRole::SwapCandidate).await;
17576    }
17577}
17578
17579#[cfg(all(test, target_os = "linux"))]
17580mod cgroup_containment_tests {
17581    use super::*;
17582    use cortexkit_test_support::ScratchDir;
17583
17584    fn running(pid: u32) -> bool {
17585        // An orphan can remain a zombie until the container init reaps it.
17586        std::fs::read_to_string(format!("/proc/{pid}/stat"))
17587            .ok()
17588            .and_then(|stat| {
17589                stat.rsplit_once(") ")
17590                    .map(|(_, rest)| rest.starts_with('Z'))
17591            })
17592            .is_some_and(|zombie| !zombie)
17593    }
17594
17595    #[tokio::test]
17596    async fn linux_teardown_reaps_the_grandchild() {
17597        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17598    }
17599
17600    #[tokio::test]
17601    async fn linux_shutdown_straggler_reaps_the_grandchild() {
17602        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17603    }
17604
17605    async fn teardown_tree(test_name: &str, shutdown: bool) {
17606        let dir = ScratchDir::new(test_name);
17607        let root = PathBuf::from(format!(
17608            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17609            std::process::id(),
17610            unix_ms_now()
17611        ));
17612        if let Err(error) = std::fs::create_dir(&root) {
17613            assert!(
17614                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17615                "required cgroup test cannot execute: {error}"
17616            );
17617            eprintln!(
17618                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17619                root.display()
17620            );
17621            return;
17622        }
17623        let placement = subc_cgroup::prepare_at(&root)
17624            .expect("prepare isolated kernel cgroup")
17625            .expect("isolated cgroup is delegated");
17626        let module_id = "tree-teardown";
17627        let module = placement
17628            .module_path(module_id)
17629            .expect("create isolated module cgroup");
17630        if !module.join("cgroup.kill").exists() {
17631            std::fs::remove_dir(&module).unwrap();
17632            std::fs::remove_dir(root.join("subc-modules")).unwrap();
17633            std::fs::remove_dir(&root).unwrap();
17634            assert!(
17635                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17636                "required cgroup.kill interface unavailable"
17637            );
17638            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17639            return;
17640        }
17641        let supervisor = Supervisor::new_for_test(
17642            Arc::new(Registry::default()),
17643            RestartPolicy::new(3, Duration::ZERO),
17644        )
17645        .with_cgroup_placement(Some(placement));
17646        let mut runtime = supervisor.runtime_config();
17647        runtime.child_roster = runtime
17648            .child_roster
17649            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17650        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17651        let pid_file = dir.join("grandchild.pid");
17652        let spec = ModuleSpec {
17653            module_id: module_id.to_string(),
17654            program: PathBuf::from("/bin/sh"),
17655            args: vec![
17656                "-c".into(),
17657                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17658                "fixture".into(),
17659                pid_file.display().to_string(),
17660            ],
17661            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17662                .into_iter()
17663                .map(|key| (key.to_string(), dir.display().to_string()))
17664                .collect(),
17665            reserved: false,
17666            reserved_prefixes: Vec::new(),
17667            protocol: ModuleProtocol::None,
17668            overlap: Default::default(),
17669        };
17670        let child =
17671            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17672        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17673        let grandchild: u32 = loop {
17674            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17675                if let Ok(pid) = contents.trim().parse() {
17676                    break pid;
17677                }
17678            }
17679            assert!(
17680                tokio::time::Instant::now() < deadline,
17681                "grandchild pid was not recorded"
17682            );
17683            tokio::time::sleep(Duration::from_millis(10)).await;
17684        };
17685        assert!(
17686            running(grandchild),
17687            "grandchild must be alive before teardown"
17688        );
17689        if shutdown {
17690            let mut child = child;
17691            crate::child_roster::end_children_for_daemon_shutdown(
17692                &runtime.child_roster,
17693                false,
17694                std::future::pending(),
17695            )
17696            .await;
17697            child.wait().await.expect("reap shutdown straggler");
17698        } else {
17699            drain_child_to_state(
17700                module_id,
17701                ModuleProtocol::None,
17702                StopNotice::NotSent,
17703                &Registry::default(),
17704                None,
17705                &snapshot,
17706                &runtime.terminal_ring,
17707                &SpawnEventFeed::default(),
17708                child,
17709                Duration::from_millis(100),
17710                ModuleState::Stopped,
17711                Some(false),
17712                &FailureContext::operator(),
17713            )
17714            .await
17715            .expect("real supervisor teardown");
17716        }
17717        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
17718        while running(grandchild) && tokio::time::Instant::now() < deadline {
17719            tokio::time::sleep(Duration::from_millis(10)).await;
17720        }
17721        let survived = running(grandchild);
17722        // Kill a surviving grandchild so a failed test does not leave it behind.
17723        if survived {
17724            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
17725            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
17726            tokio::time::sleep(Duration::from_millis(100)).await;
17727        }
17728        if module.exists() {
17729            std::fs::remove_dir(&module).expect("remove empty module cgroup");
17730        }
17731        std::fs::remove_dir(root.join("subc-modules")).unwrap();
17732        std::fs::remove_dir(&root).unwrap();
17733        assert!(
17734            !survived,
17735            "grandchild {grandchild} outlived module teardown"
17736        );
17737        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
17738    }
17739}
17740
17741/// A slow live-children record write on the restart path must not stop the
17742/// daemon from answering the replacement module's authentication handshake.
17743#[cfg(all(test, unix))]
17744mod record_write_stall_tests {
17745    use super::*;
17746    use std::sync::atomic::{AtomicBool, Ordering};
17747    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
17748
17749    /// `subc-client-rs` gives the whole handshake this long before the module
17750    /// exits with `Timeout { stage: ServerProof }`.
17751    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
17752    /// How long the replacement's admission write is held: longer than the
17753    /// module's deadline, standing in for the delay a busy disk can add
17754    /// across the two syncs in that write.
17755    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17756
17757    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
17758    /// On macOS every module is launched through the privacy trampoline, and
17759    /// a module that is not a `#!` script is admitted only if its running
17760    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
17761    /// the shell soon replaces itself with `/bin/sleep`, which does not match
17762    /// the configured `/bin/sh`. The image is read only after the spawn's
17763    /// live-children record write, and here that write lasted long enough for
17764    /// the shell to have exec'd every time, so each process was refused and
17765    /// killed right after spawn, and the supervisor crash-looped it with
17766    /// backoff until its restart budget ran out. The restart this test
17767    /// measures would then never produce a module at all.
17768    fn sleeping_spec() -> ModuleSpec {
17769        ModuleSpec {
17770            module_id: "record-write-stall".into(),
17771            program: "/bin/sleep".into(),
17772            args: vec!["600".into()],
17773            env: vec![],
17774            reserved: false,
17775            reserved_prefixes: vec![],
17776            protocol: ModuleProtocol::None,
17777            overlap: Default::default(),
17778        }
17779    }
17780
17781    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17782    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17783        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17784        let supervisor =
17785            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17786                .with_live_children_record(home.join("live-children.json"));
17787        let module = supervisor.spawn(sleeping_spec()).unwrap();
17788        // `reported_pid` names a process only once it is confirmed as the
17789        // module (on macOS, once the trampoline's image check has passed), so
17790        // a process the daemon is about to refuse and kill never counts as
17791        // the running module, here or in the final wait below.
17792        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17793            loop {
17794                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17795                    .unwrap()
17796                    .reported_pid()
17797                {
17798                    return pid;
17799                }
17800                tokio::time::sleep(Duration::from_millis(10)).await;
17801            }
17802        })
17803        .await
17804        .expect("the first process was admitted as the module");
17805
17806        // The daemon's listener, served by this runtime's workers like the
17807        // supervisor's own tasks.
17808        let key = vec![0x42; 32];
17809        let daemon_id = [0x24; 16];
17810        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17811        let port = listener.local_addr().unwrap().port();
17812        let server = tokio::spawn(crate::server::serve_listener(
17813            listener,
17814            Arc::new(crate::router::Router::with_default_self_handler()),
17815            crate::server::ServerAuth::with_limits(
17816                key.clone(),
17817                daemon_id,
17818                "test",
17819                crate::server::DEFAULT_AUTH_DEADLINE,
17820                4,
17821            ),
17822        ));
17823
17824        // Hold the write that admits the replacement, once, and say when it
17825        // has started.
17826        let (held_tx, held_rx) = std::sync::mpsc::channel();
17827        let armed = AtomicBool::new(true);
17828        supervisor
17829            .child_roster
17830            .set_record_write_hook_for_test(move |live| {
17831                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17832                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17833                    let _ = held_tx.send(());
17834                    std::thread::sleep(HELD_RECORD_WRITE);
17835                }
17836            });
17837
17838        // The replacement module, as a separate process would be: its own
17839        // thread and runtime, so only the daemon's runtime can be stalled.
17840        let module_process = std::thread::spawn(move || {
17841            held_rx
17842                .recv_timeout(Duration::from_secs(30))
17843                .expect("the restart reached the replacement's record write");
17844            let conn = ConnectionInfo {
17845                schema: SCHEMA_VERSION,
17846                wire_version: None,
17847                endpoints: vec![Endpoint {
17848                    host: "127.0.0.1".to_owned(),
17849                    port,
17850                }],
17851                key,
17852                daemon_id,
17853                pid: std::process::id(),
17854                daemon_ver: "test".to_owned(),
17855            };
17856            tokio::runtime::Builder::new_current_thread()
17857                .enable_all()
17858                .build()
17859                .unwrap()
17860                .block_on(async {
17861                    let started = Instant::now();
17862                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17863                        .await
17864                        .unwrap();
17865                    let result =
17866                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17867                    (result, started.elapsed())
17868                })
17869        });
17870
17871        module.restart(Some(0)).await.unwrap();
17872        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17873            .await
17874            .unwrap();
17875        assert!(
17876            auth.is_ok(),
17877            "the daemon must answer a module's handshake while a restart waits on the \
17878             live-children record write; got {auth:?} after {elapsed:?}"
17879        );
17880
17881        tokio::time::timeout(Duration::from_secs(20), async {
17882            while lock_snapshot(&module.inner.snapshot)
17883                .unwrap()
17884                .reported_pid()
17885                .is_none_or(|pid| pid == old_pid)
17886            {
17887                tokio::time::sleep(Duration::from_millis(10)).await;
17888            }
17889        })
17890        .await
17891        .expect("the restart published its replacement");
17892        module.stop().await.unwrap();
17893        server.abort();
17894    }
17895}