Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68/// Command-line flag used by supervised modules to find subc.
69///
70/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
71/// The path points at the TCP+key connection file; it is not an ambient signal and
72/// is never inherited by standalone children.
73pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
79/// to contain a real crash loop (which respawns in seconds) and short enough
80/// that unrelated crashes hours apart never accumulate into a permanent stop.
81const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82/// How long a drain waits for already-dispatched requests to finalize before
83/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
84/// not RPC-scale: the original 2s value silently cut nearly every real tool
85/// call at the fence, making the wait-for-finalize design decorative for the
86/// workloads it existed for. Quiescence short-circuits, so an idle module
87/// restarts immediately regardless of this value; the budget is spent only
88/// when a genuine in-flight request is worth finishing. Per-module override:
89/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
90/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
91/// where a stuck request will never settle).
92pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96/// How long a drain waits after one busy-gauge probe completes before it asks
97/// again.
98///
99/// A drain re-probes on this short interval of its own rather than on the
100/// module's health cadence. The default cadence (30 s) equals the default
101/// drain ceiling, so a cadence-paced drain that found the module busy at its
102/// first probe never asked a second time: it waited out the whole ceiling and
103/// forced GOODBYE even after the gauges had read zero. The interval is counted
104/// from the moment the previous probe finished, and probes run one at a time,
105/// so a slow module is never asked more often than it can answer. Periodic
106/// health checks outside a drain keep using the configured cadence.
107const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108/// How long a restart waits for the exited process's output readers.
109///
110/// The restart does not depend on either output reader finishing. A stderr
111/// reader still running at this bound is left running, and its later output goes
112/// into the exited process's own section of the stderr ring (see
113/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
114/// bound no longer decides whether a crash's last lines are kept: under load
115/// the reader may simply not have been scheduled yet, and cutting it there
116/// discarded exactly the lines that explained the crash.
117///
118/// What the bound still decides is when the tail starts reporting
119/// `Incomplete`: a pipe open past it usually means a descendant of the exited
120/// process still holds it, and the tail cannot claim to be whole until that
121/// pipe closes. That is also the only case in which the wait costs the restart
122/// anything, because a pipe with no other holder reaches EOF when the process
123/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
124/// `Captured` at EOF with nothing lost.
125///
126/// The stdout reader carries no ring, only the capture file. It also continues
127/// to EOF after this bound: child exit closes the writer but does not mean the
128/// reader has consumed all buffered bytes. Late output from either pipe can
129/// therefore reach the shared capture file after the successor starts.
130const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
132pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
135/// dropped because the subscriber stopped draining its frames. The detail's
136/// `first_undelivered_cursor` names the first event it did not receive; the
137/// client resubscribes from the last cursor it did receive.
138pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143    deadline: tokio::time::Instant,
144    expected: Option<subc_os::FileIdentity>,
145    trampoline: Option<subc_os::FileIdentity>,
146    script: bool,
147    module_id: String,
148    #[cfg(test)]
149    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
155    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
156    let path = std::env::current_exe()
157        .unwrap()
158        .parent()
159        .unwrap()
160        .parent()
161        .unwrap()
162        .join("privacy-trampoline-fixture");
163    // Without the fixture every macOS spawn is refused, and the tests that
164    // spawn fail later as a module in state Failed, which names the wrong
165    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
166    assert!(
167        path.exists(),
168        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169         --bins --features test-support` or `cargo test -p subc-daemon` first",
170        path.display()
171    );
172    path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177    use std::io::Read;
178    let mut probe = std::process::Command::new(path)
179        .args(["__disclaim-exec", "--probe"])
180        .stdin(Stdio::null())
181        .stdout(Stdio::piped())
182        .stderr(Stdio::piped())
183        .spawn()
184        .map_err(|error| {
185            format!(
186                "privacy trampoline probe failed for {}: {error}",
187                path.display()
188            )
189        })?;
190    let deadline = std::time::Instant::now() + Duration::from_secs(5);
191    let status = loop {
192        match probe.try_wait() {
193            Ok(Some(status)) => break status,
194            Ok(None) if std::time::Instant::now() < deadline => {
195                std::thread::sleep(Duration::from_millis(5))
196            }
197            result => {
198                let _ = probe.kill();
199                let _ = probe.wait();
200                return Err(format!(
201                    "privacy trampoline probe failed or timed out for {}: {result:?}",
202                    path.display()
203                ));
204            }
205        }
206    };
207    let mut answer = String::new();
208    if let Some(stdout) = probe.stdout.take() {
209        let _ = stdout.take(256).read_to_string(&mut answer);
210    }
211    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212        return Ok(());
213    }
214    let mut diagnostic = String::new();
215    if let Some(stderr) = probe.stderr.take() {
216        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217    }
218    let cause = diagnostic
219        .trim()
220        .strip_prefix("ck-subc: own privacy identity refused: ")
221        .unwrap_or("binary does not implement the privacy trampoline protocol");
222    Err(format!(
223        "{cause}: probe of {} exited {status}",
224        path.display()
225    ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230    spec: &ModuleSpec,
231    roster: &ChildRoster,
232) -> Result<
233    (
234        Command,
235        Option<PrivacyExec>,
236        subc_os::privacy_identity::ExecAcknowledgement,
237    ),
238    SuperviseError,
239> {
240    let failure = |cause: String| {
241        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242        SuperviseError::Spawn {
243            program: spec.program.clone(),
244            source: io::Error::other(cause),
245            cgroup_path: None,
246        }
247    };
248    let trampoline = roster.privacy_trampoline().map_err(failure)?;
249    // Resolve PATH with the same environment the Command will receive. For
250    // scripts retain the existing orphan-identity rule: the kernel chooses
251    // the interpreter, and its observed image is the one recorded. Do not
252    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
253    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254        let path = spec
255            .env
256            .iter()
257            .find(|(key, _)| key == "PATH")
258            .map(|(_, value)| std::ffi::OsString::from(value))
259            .or_else(|| std::env::var_os("PATH"))
260            .unwrap_or_else(|| "/usr/bin:/bin".into());
261        std::env::split_paths(&path)
262            .map(|dir| dir.join(&spec.program))
263            .find(|path| path.is_file())
264            .unwrap_or_else(|| spec.program.clone())
265    } else {
266        spec.program.clone()
267    };
268    let expected = subc_os::file_identity(&program);
269    let trampoline_image = subc_os::file_identity(&trampoline);
270    let script = {
271        use std::io::Read;
272        let mut prefix = [0u8; 2];
273        std::fs::File::open(&program)
274            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275    };
276    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277        return Err(failure(
278            "privacy identity module executable is missing or is the trampoline itself".to_string(),
279        ));
280    }
281    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282        .map_err(|error| failure(error.to_string()))?;
283    let reader =
284        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285    let mut command = Command::new(&trampoline);
286    command
287        .arg("__disclaim-exec")
288        .arg(ack.fd().to_string())
289        .arg(&program);
290    ack.install(command.as_std_mut());
291    Ok((
292        command,
293        Some(PrivacyExec {
294            reader,
295            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296            expected,
297            trampoline: trampoline_image,
298            script,
299            module_id: spec.module_id.clone(),
300            #[cfg(test)]
301            confirmation_pause: spec
302                .env
303                .iter()
304                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305                .map(|(_, address)| (address.clone(), None)),
306        }),
307        ack,
308    ))
309}
310
311struct SupervisedChild {
312    subtree_target: crate::subtree_resources::Target,
313    child: Child,
314    exec: crate::activity::ExecOwner,
315    #[cfg(windows)]
316    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
317    #[cfg(target_os = "macos")]
318    privacy_exec: Option<PrivacyExec>,
319    /// Set once this launch's exec acknowledgement confirms the module image.
320    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
321    /// `subc_os::privacy_identity`), which then replaces itself with the
322    /// module. The supervisor owns and can kill that pid from spawn, but
323    /// status readers report it only after this latch is set, so nothing
324    /// reports the trampoline's image as the module's.
325    #[cfg(target_os = "macos")]
326    report_ready: Arc<OnceLock<()>>,
327    /// Supervisor failure detail retained for the terminal record, including
328    /// image refusals and first-HELLO timeouts after a successful spawn.
329    spawn_failure: Option<String>,
330    /// The protocol this process was launched with. A reload can store a new
331    /// launch spec with a different protocol, but that takes effect only at the
332    /// next spawn, so this process keeps being handled by the protocol it
333    /// actually speaks.
334    protocol: ModuleProtocol,
335    /// This process's cgroup name: a bounded module/slot label followed by a
336    /// spawn suffix unique to this process (when cgroup placement is on). A
337    /// retired process in a slot may still be draining when a later one is
338    /// spawned into that slot, so the suffix keeps the later process out of
339    /// the retired one's cgroup, which is the domain a kill applies to.
340    #[cfg(target_os = "linux")]
341    module_id: String,
342    #[cfg(target_os = "linux")]
343    cgroup_placement: Option<subc_cgroup::Placement>,
344    /// The job that contains this child and every process it spawns (issue #109).
345    ///
346    /// Dropping this handle is what reaps a surviving tree when no supervisor
347    /// code runs — a daemon crash — because the job carries
348    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
349    ///
350    /// That limit is not crash-only, and the difference is worth knowing: a
351    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
352    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
353    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
354    /// module at once. Before this change they survived that, saw EOF on the
355    /// control socket, and ran their own teardown; Unix keeps that path
356    /// deliberately, so a module can seal a WAL or close a capture rather than
357    /// be killed mid-write. So this trades graceful teardown on every Windows
358    /// daemon stop for containment on a crash, which is the right way round
359    /// today: orphaned GPU workers are a reported, recurring problem, and the
360    /// modules that write most heavily do not run on Windows.
361    ///
362    /// The fix is a real Windows stop path — the daemon draining before it
363    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
364    /// reaches only what the drain left behind, which is what it should reach.
365    #[cfg(windows)]
366    job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
367    stdout_pump: Option<JoinHandle<()>>,
368    stderr_pump: Option<StderrPump>,
369    stderr_ring: Arc<Mutex<StderrRing>>,
370    /// Conservative monotonic bound sampled before `command.spawn()`, not when
371    /// identity recording finishes or the spawn generation is published.
372    spawned_at: Instant,
373    /// Ordinary spawns must send their first HELLO within this budget. Once
374    /// accepted, a later disconnect must not restart the registration clock.
375    registration_deadline: Option<Instant>,
376    registration_watch: Option<RegistrationWatch>,
377    spawned_at_ms: u64,
378    spawned_from: PathBuf,
379    launch_spec_digest: LaunchSpecDigest,
380    spawned_file_identity: Option<SpawnedFileIdentity>,
381    process_start_time: Option<u64>,
382    process_identity: Option<ProcessIdentity>,
383    pid: u32,
384    /// This process's entry in the daemon's child roster, released when the
385    /// process is reaped or this handle is dropped.
386    roster_guard: Option<crate::child_roster::RosterGuard>,
387}
388
389impl SupervisedChild {
390    fn registered_since_spawn(&self) -> bool {
391        self.registration_watch
392            .as_ref()
393            .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
394    }
395
396    fn id(&self) -> Option<u32> {
397        Some(self.pid)
398    }
399
400    fn process_identity(&self) -> Option<ProcessIdentity> {
401        self.process_identity
402    }
403
404    async fn wait(&mut self) -> io::Result<ExitStatus> {
405        #[cfg(target_os = "macos")]
406        self.confirm_privacy_exec().await;
407        // The roster entry is NOT released here. A daemon shutdown waits for the
408        // roster to empty and then exits the process, so releasing at the reap
409        // let it exit before the exit handler wrote this child's terminal record
410        // (the stderr drain and snapshot update sit in between), and the
411        // shutdown's own `daemon_shutdown` record was intermittently lost. The
412        // caller releases it after recording the exit (`release_roster`), and
413        // dropping the handle releases it too.
414        let result = self.child.wait().await;
415        #[cfg(target_os = "linux")]
416        if result.is_ok() {
417            if let Some(placement) = self.cgroup_placement.as_ref() {
418                cleanup_reaped_cgroup(placement, &self.module_id).await;
419                // Keep ownership while awaiting kernel population changes: a
420                // drain timeout may cancel this wait and then escalate/reap.
421                self.cgroup_placement = None;
422            }
423        }
424        result
425    }
426
427    #[cfg(target_os = "macos")]
428    async fn confirm_privacy_exec(&mut self) {
429        let Some(pending) = &mut self.privacy_exec else {
430            return;
431        };
432        let result = tokio::time::timeout_at(pending.deadline, async {
433            let mut record = Vec::new();
434            loop {
435                let mut ready = pending.reader.readable().await?;
436                let read = ready.try_io(|reader| {
437                    use std::io::Read;
438                    let mut reader = reader.get_ref();
439                    let mut buffer = [0u8; 256];
440                    reader.read(&mut buffer).map(|count| (count, buffer))
441                });
442                match read {
443                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
444                    Ok(Ok((count, buffer))) => {
445                        if record.len() + count > 1024 {
446                            return Err(io::Error::other(
447                                "privacy exec refusal record is too long",
448                            ));
449                        }
450                        record.extend_from_slice(&buffer[..count]);
451                    }
452                    Ok(Err(error)) => return Err(error),
453                    Err(_) => continue,
454                }
455            }
456        })
457        .await;
458        #[cfg(test)]
459        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
460            use tokio::io::{AsyncReadExt, AsyncWriteExt};
461            if stream.is_none() {
462                *stream = Some(
463                    tokio::net::TcpStream::connect(address.as_str())
464                        .await
465                        .unwrap(),
466                );
467                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
468                stream
469                    .as_mut()
470                    .unwrap()
471                    .write_all(&self.pid.to_ne_bytes())
472                    .await
473                    .unwrap();
474            }
475            let mut reply = [0];
476            stream
477                .as_mut()
478                .unwrap()
479                .read_exact(&mut reply)
480                .await
481                .unwrap();
482            pending.confirmation_pause = None;
483            if &reply == b"F" {
484                Ok(Err(io::Error::other("injected late exec refusal")))
485            } else {
486                assert_eq!(&reply, b"X");
487                result
488            }
489        } else {
490            result
491        };
492        // Keep the reader in self across await: select cancellation must not
493        // discard the handshake or reset its original five-second deadline.
494        let pending = self.privacy_exec.as_ref().expect("pending exec");
495        let cause = match result {
496            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
497            Ok(Err(error)) => Some(format!(
498                "privacy identity exec acknowledgement failed: {error}"
499            )),
500            Ok(Ok(record)) if !record.is_empty() => Some(
501                std::str::from_utf8(&record)
502                    .ok()
503                    .and_then(|record| {
504                        record
505                            .trim()
506                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
507                    })
508                    .filter(|cause| !cause.is_empty())
509                    .unwrap_or("invalid privacy exec refusal record")
510                    .to_string(),
511            ),
512            Ok(Ok(_)) => match self.child.try_wait() {
513                // Empty EOF is the exec acknowledgement. A real module may exit
514                // immediately, including with a reserved trampoline status; no
515                // image is admitted, and its ordinary exit contract stays intact.
516                Ok(Some(_status)) => None,
517                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
518                Ok(None) => {
519                    let image = observe_spawned_image(self.pid);
520                    if let Some(image) = image.filter(|image| {
521                        image.executable.is_some()
522                            && image.executable != pending.trampoline
523                            && (image.executable == pending.expected || pending.script)
524                    }) {
525                        if let Err(error) = self
526                            .roster_guard
527                            .as_ref()
528                            .map(|guard| guard.confirm_executable(image))
529                            .transpose()
530                        {
531                            Some(format!(
532                                "could not durably record confirmed module image: {error}"
533                            ))
534                        } else {
535                            let _ = self.report_ready.set(());
536                            info!(module_id = %pending.module_id, pid = self.pid,
537                                "module spawned with own privacy identity (responsibility disclaimed)");
538                            None
539                        }
540                    } else if image.is_none()
541                        || image.is_some_and(|image| image.executable.is_none())
542                    {
543                        // A process can exit between try_wait and the kernel
544                        // image read. Empty EOF already acknowledged exec, so
545                        // preserve that module's ordinary exit rather than
546                        // mislabel a disappearing image as trampoline refusal.
547                        // Keep pending in self across await so cancellation does
548                        // not discard validation or reset its original deadline.
549                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
550                            Ok(Ok(_status)) => None,
551                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
552                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
553                        }
554                    } else {
555                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
556                    }
557                }
558            },
559        };
560        let pending = self.privacy_exec.take().expect("pending exec");
561        // Exec confirmation describes launch success, not roster retirement.
562        // Keep even a reaped child recorded until its terminal history exists.
563        self.exec.0.finish(if cause.is_some() {
564            crate::activity::ExecState::Refused
565        } else {
566            crate::activity::ExecState::CommittedExec
567        });
568        if let Some(cause) = cause {
569            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
570            self.spawn_failure = Some(cause);
571            // No image is admitted on failure. Reach the entire fresh process
572            // group, including a module which spawned a helper before refusal.
573            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
574                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
575            }
576            let _ = self.child.start_kill();
577        }
578    }
579
580    /// Releases this child's daemon-shutdown roster entry once its exit has
581    /// been recorded. The pid is already reaped and free for reuse, so the
582    /// entry must not outlive the record any longer than that.
583    fn release_roster(&mut self) {
584        self.roster_guard = None;
585    }
586
587    /// Kill the child and, where containment is available, its process tree.
588    ///
589    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
590    /// helper process leaked the helper — the Synapse embedding module's CUDA
591    /// worker holds the GPU allocation, so the leak cost VRAM until the next
592    /// restart of something else. Terminating the job reaches grandchildren that
593    /// a tree walk cannot, including one whose parent has already exited and
594    /// been reparented away.
595    ///
596    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
597    /// direct-child kill still decides the outcome, so containment can never
598    /// change whether a module is reported as stopped.
599    fn start_kill(&mut self) -> io::Result<()> {
600        #[cfg(windows)]
601        if let Some(job) = &self.job {
602            if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
603                debug!(
604                    error = %error,
605                    "job termination failed; the direct-child kill still owns the outcome"
606                );
607            }
608        }
609        #[cfg(target_os = "linux")]
610        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
611        self.child.start_kill()
612    }
613
614    async fn drain_stderr(&mut self, module_id: &str) {
615        if let Some(pump) = self.stdout_pump.take() {
616            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
617        }
618
619        let Some(pump) = self.stderr_pump.take() else {
620            return;
621        };
622        settle_stderr_pump(
623            module_id,
624            &self.stderr_ring,
625            pump,
626            STDERR_PUMP_DRAIN_TIMEOUT,
627        )
628        .await;
629    }
630}
631
632/// Bound the restart's wait, not the lifetime of the exited child's stdout
633/// reader. Slow pipe reads or capture writes must not turn the bound into lost
634/// output; dropping the handle lets the reader finish at EOF, like stderr.
635async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
636    match timeout(bound, &mut pump).await {
637        Ok(Ok(())) => {}
638        Ok(Err(error)) => {
639            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
640        }
641        Err(_) => {
642            drop(pump);
643            warn!(
644                module_id,
645                waited = ?bound,
646                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
647            );
648        }
649    }
650}
651
652/// The reader task for one process's stderr, with the ring generation its
653/// lines are attributed to.
654struct StderrPump {
655    task: JoinHandle<()>,
656    generation: u64,
657}
658
659/// Retire an exited process's stderr reader and wait up to `bound` for it to
660/// reach EOF. A reader still running at the bound is detached, not stopped: it
661/// keeps filling the exited process's section of the ring until its pipe
662/// closes, and the tail reads `Incomplete` until then. See
663/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
664async fn settle_stderr_pump(
665    module_id: &str,
666    ring: &Arc<Mutex<StderrRing>>,
667    pump: StderrPump,
668    bound: Duration,
669) {
670    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
671    let StderrPump {
672        mut task,
673        generation,
674    } = pump;
675    lock().retire_pump(generation);
676    match timeout(bound, &mut task).await {
677        Ok(Ok(())) => {}
678        Ok(Err(err)) => {
679            let mut ring = lock();
680            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
681            ring.finish_pump(generation);
682            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
683        }
684        Err(_) => {
685            // Dropping the handle detaches the task; it ends at EOF on its pipe.
686            drop(task);
687            lock().mark_pump_late(
688                generation,
689                format!(
690                    "stderr of the exited process had not reached EOF {bound:?} after it was \
691                     retired (a descendant may still hold the pipe open); lines it still \
692                     writes are kept in that process's section"
693                ),
694            );
695            warn!(
696                module_id,
697                waited = ?bound,
698                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
699            );
700        }
701    }
702}
703
704fn registration_release_events() -> &'static watch::Sender<u64> {
705    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
706    EVENTS.get_or_init(|| {
707        let (sender, _receiver) = watch::channel(0);
708        sender
709    })
710}
711
712pub(crate) fn notify_registration_release() {
713    let events = registration_release_events();
714    let next_generation = (*events.borrow()).wrapping_add(1);
715    events.send_replace(next_generation);
716}
717
718/// How to launch one singleton module process.
719#[derive(Debug, Clone, PartialEq, Eq)]
720pub struct ModuleSpec {
721    pub module_id: String,
722    pub program: PathBuf,
723    pub args: Vec<String>,
724    pub env: Vec<(String, String)>,
725    /// When true this is a reserved module: each spawn gets a fresh one-time launch
726    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
727    /// process can register this module_id (a security-boundary module like the
728    /// credential vault must not be impersonable while it is down/restarting).
729    pub reserved: bool,
730    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
731    /// Prefixes come from daemon config and must end in `:` before they reach the
732    /// supervisor; the owner module's current spawn nonce authorizes claims under
733    /// each prefix.
734    pub reserved_prefixes: Vec<String>,
735    /// The wire protocol this module speaks, as DECLARED in daemon config.
736    ///
737    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
738    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
739    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
740    /// and NO launch nonce, and a clean exit the daemon did not request is
741    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
742    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
743    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
744    /// because a process ignores an environment variable it does not read.
745    ///
746    /// The argument is the part that cannot be "harmless to a process that
747    /// ignores it": a stock binary exits on an unknown flag before it listens
748    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
749    /// first conformance run against this mode found it. The nonce is withheld
750    /// because a process that will never present it gains nothing from holding
751    /// it, and a secret in the environment of a process that does not need it is
752    /// a leak surface for no benefit.
753    pub protocol: ModuleProtocol,
754    /// Whether two processes of this module may run at once, which is what a
755    /// blue/green swap does for the length of its overlap. Declared in daemon
756    /// config because the daemon must be able to answer it while the module is
757    /// down, and so a module cannot talk itself into it after registering.
758    pub overlap: ModuleOverlap,
759}
760
761/// Only the configured launch inputs enter this value; spawned connection paths,
762/// nonces and inherited environment never do. Its bytes remain daemon-private.
763#[derive(Clone, PartialEq, Eq)]
764pub(crate) struct LaunchSpecDigest([u8; 32]);
765
766impl fmt::Debug for LaunchSpecDigest {
767    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
768        f.write_str("LaunchSpecDigest([redacted])")
769    }
770}
771
772impl LaunchSpecDigest {
773    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
774        fn field(hash: &mut Sha256, bytes: &[u8]) {
775            hash.update((bytes.len() as u64).to_le_bytes());
776            hash.update(bytes);
777        }
778        let mut hash = Sha256::new();
779        if let Some(program) = spec.program.to_str() {
780            field(&mut hash, b"utf8-program");
781            field(&mut hash, program.as_bytes());
782        } else {
783            // The daemon config is UTF-8, but in-process callers may provide
784            // native paths: lossy conversion would collapse distinct programs.
785            #[cfg(unix)]
786            {
787                use std::os::unix::ffi::OsStrExt;
788                field(&mut hash, b"unix-program");
789                field(&mut hash, spec.program.as_os_str().as_bytes());
790            }
791            #[cfg(windows)]
792            {
793                use std::os::windows::ffi::OsStrExt;
794                field(&mut hash, b"windows-program");
795                let bytes = spec
796                    .program
797                    .as_os_str()
798                    .encode_wide()
799                    .flat_map(u16::to_le_bytes)
800                    .collect::<Vec<_>>();
801                field(&mut hash, &bytes);
802            }
803            #[cfg(not(any(unix, windows)))]
804            field(&mut hash, spec.program.to_string_lossy().as_bytes());
805        }
806        hash.update((spec.args.len() as u64).to_le_bytes());
807        for arg in &spec.args {
808            field(&mut hash, arg.as_bytes());
809        }
810        let mut env = spec.env.iter().collect::<Vec<_>>();
811        env.sort_unstable();
812        hash.update((env.len() as u64).to_le_bytes());
813        for (key, value) in env {
814            field(&mut hash, key.as_bytes());
815            field(&mut hash, value.as_bytes());
816        }
817        field(
818            &mut hash,
819            match spec.protocol {
820                ModuleProtocol::Subc => b"subc",
821                ModuleProtocol::None => b"none",
822            },
823        );
824        Self(hash.finalize().into())
825    }
826}
827
828/// Whether a module tolerates a second process of itself running alongside.
829///
830/// Most modules are single-writer on their store (a WAL, a capture log, a
831/// resident index behind a writer barrier), and two processes on one store
832/// corrupt it. So a swap, which overlaps the old and new process by design,
833/// is refused unless the module's config opts in with `overlap: "safe"`.
834#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
835pub enum ModuleOverlap {
836    /// Never run two processes of this module at once. The default.
837    #[default]
838    Exclusive,
839    /// The module has said a second process of itself is harmless for the
840    /// length of a swap.
841    ///
842    /// Declare it only if a second instance can run for a few seconds without
843    /// touching ANY single-writer store: every database, WAL, index, projector
844    /// and scheduled job the module owns. A lease on part of that state is not
845    /// enough. broca's session lease guards WAL appends while its run index, its
846    /// store projector and its archive fold timer (which unlinks live WAL files)
847    /// stay single-writer, so broca is exclusive despite holding a lease. The
848    /// refusal only fires after this has been decided, so the decision is the
849    /// check.
850    Safe,
851}
852
853impl ModuleOverlap {
854    pub fn as_str(self) -> &'static str {
855        match self {
856            Self::Exclusive => "exclusive",
857            Self::Safe => "safe",
858        }
859    }
860}
861
862/// Environment variable telling a spawned module which case it was started
863/// for, before it sends HELLO. Only a swap candidate carries it, as
864/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
865///
866/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
867/// longer because nobody waits on it, while a plain restart must flip ready
868/// quickly because callers see `module_warming` until it does. Absence means
869/// plain restart, the safe reading. The daemon trusts nothing about it; the
870/// candidate is proven by its launch nonce at HELLO.
871pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
872/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
873pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
874/// How long a swap waits for its candidate to register and declare itself
875/// ready when the operator does not say. A module warming as a swap candidate
876/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
877/// daemon allows that plus time to start the process and send HELLO.
878pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
879
880/// Bounded restart policy for crash exits.
881///
882/// `max_restarts` is the number of replacement processes allowed after the
883/// initial spawn WITHIN `window`. After that many crash restarts inside one
884/// window the module enters [`ModuleState::Failed`] and the supervisor stops
885/// the crash loop.
886///
887/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
888/// and that only survived because crashes were rare: a module that crashed
889/// three times across a week was disabled forever by crashes that had nothing
890/// to do with each other. That stopped being survivable once modules began
891/// exiting non-zero whenever the daemon's connection to them drops, because
892/// then every daemon-side connection drop spends a unit of the same budget and
893/// one flappy hour permanently stops a healthy module. Restarts older than
894/// `window` release their slot, so a module that crashed twice yesterday has a
895/// full budget today, while a genuine crash loop -- which is fast by
896/// definition -- still reaches the cap and stops.
897#[derive(Debug, Clone, Copy, PartialEq, Eq)]
898pub struct RestartPolicy {
899    pub max_restarts: u32,
900    /// Base delay before a crash replacement. The actual delay escalates with
901    /// the number of recent crash replacements and is capped by `max_backoff`.
902    pub backoff: Duration,
903    /// Maximum delay before a crash replacement.
904    pub max_backoff: Duration,
905    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
906    /// budget effectively infinite (nothing is ever in-window), which is why
907    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
908    pub window: Duration,
909}
910
911impl RestartPolicy {
912    /// A policy with the default crash window. Callers that care about the
913    /// window say so with [`Self::with_window`]; the ones that do not are
914    /// asking for the standard rate limit, not for no limit.
915    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
916        Self {
917            max_restarts,
918            backoff,
919            max_backoff: DEFAULT_MAX_BACKOFF,
920            window: DEFAULT_RESTART_WINDOW,
921        }
922    }
923
924    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
925        self.max_backoff = max_backoff;
926        self
927    }
928
929    pub fn with_window(mut self, window: Duration) -> Self {
930        self.window = window;
931        self
932    }
933
934    /// Calculate the capped exponential delay for the next crash replacement.
935    /// `restart_in_window` is zero for the first replacement after an operator
936    /// action (restart, reload, re-enable) cleared the crash ring, or after all
937    /// older crash replacements have aged out of the window.
938    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
939        if self.backoff.is_zero() || self.max_backoff.is_zero() {
940            return Duration::ZERO;
941        }
942
943        let mut delay = self.backoff;
944        for _ in 0..restart_in_window {
945            if delay >= self.max_backoff {
946                return self.max_backoff;
947            }
948            delay = delay
949                .checked_mul(10)
950                .unwrap_or(self.max_backoff)
951                .min(self.max_backoff);
952        }
953        delay.min(self.max_backoff)
954    }
955
956    /// The one sentence that explains a budget-exhausted stop, used for both the
957    /// log line and the terminal record so the two cannot drift. It names the
958    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
959    /// exactly what this budget is not.
960    fn budget_exhausted_detail(&self) -> String {
961        format!(
962            "crash budget exhausted: max_restarts={} within window_secs={}",
963            self.max_restarts,
964            self.window.as_secs()
965        )
966    }
967}
968
969impl Default for RestartPolicy {
970    fn default() -> Self {
971        Self {
972            max_restarts: DEFAULT_MAX_RESTARTS,
973            backoff: DEFAULT_BACKOFF,
974            max_backoff: DEFAULT_MAX_BACKOFF,
975            window: DEFAULT_RESTART_WINDOW,
976        }
977    }
978}
979
980#[derive(Debug, Clone, Copy, PartialEq, Eq)]
981struct CrashRestartSchedule {
982    restart_in_window: u32,
983    delay: Duration,
984}
985
986/// Whether the daemon itself will bring this module back after the exit being
987/// handled: it is enabled AND its in-window crash restarts are below the cap.
988///
989/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
990/// the window are dropped here rather than by a timer, so the count is right
991/// the moment somebody asks and no bookkeeping runs for idle modules.
992fn daemon_will_restart(
993    state: &mut SupervisorSnapshot,
994    policy: &RestartPolicy,
995    now: Instant,
996) -> bool {
997    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
998}
999
1000const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1001const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1002const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1003const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1004
1005#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1006pub enum HealthAction {
1007    Report,
1008    Restart,
1009    Alert,
1010}
1011
1012impl fmt::Display for HealthAction {
1013    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1014        f.write_str(match self {
1015            Self::Report => "report",
1016            Self::Restart => "restart",
1017            Self::Alert => "alert",
1018        })
1019    }
1020}
1021
1022#[derive(Debug, Clone, PartialEq, Eq)]
1023pub struct HealthConfig {
1024    /// Optional loopback HTTP endpoint for a managed non-wire process.
1025    /// Changing it applies live on rescan; the process protocol changes only
1026    /// at its next spawn.
1027    pub http: Option<String>,
1028    pub cadence: Duration,
1029    pub deadline: Duration,
1030    pub failure_threshold: u32,
1031    pub on_degraded: HealthAction,
1032    pub on_failing: HealthAction,
1033    pub critical: bool,
1034}
1035
1036impl Default for HealthConfig {
1037    fn default() -> Self {
1038        Self {
1039            http: None,
1040            cadence: DEFAULT_HEALTH_CADENCE,
1041            deadline: DEFAULT_HEALTH_DEADLINE,
1042            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1043            on_degraded: HealthAction::Report,
1044            on_failing: HealthAction::Report,
1045            critical: false,
1046        }
1047    }
1048}
1049
1050/// The supervisor's view of one module's health, relayed to clients over
1051/// channel-0 and rendered by `ck health`.
1052///
1053/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1054/// stated here rather than only at the wire type a consumer reads. A reader can
1055/// look up what `None` means; only a writer can silently change it, and the
1056/// writer has no reason to go looking at a downstream contract before editing.
1057///
1058/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1059/// back to `None` on re-registration precisely so a respawned module does not
1060/// carry its predecessor's timestamp — so an old value and an absent one call for
1061/// opposite readings, and anything that defaulted this to a number would make a
1062/// never-probed module indistinguishable from one probed at the epoch.
1063///
1064/// `detail` and `metrics` are `None` when the module published none on this
1065/// probe, which does not mean it reported nothing wrong — it is also the shape
1066/// when the probe never reached it. `last_probe_ms` is what separates those.
1067#[derive(Debug, Clone, PartialEq)]
1068pub struct ModuleHealthStatus {
1069    pub status: SupervisorHealthStatus,
1070    pub last_probe_ms: Option<u64>,
1071    pub detail: Option<String>,
1072    pub metrics: Option<Value>,
1073    pub consecutive_failures: u32,
1074    /// Number of replies received after a recurring health probe's deadline.
1075    /// Unlike a timeout, every increment proves the module was alive.
1076    pub late_answer_count: u64,
1077    /// End-to-end latency of the newest late reply, measured from probe start.
1078    pub last_late_answer_latency_ms: Option<u64>,
1079    pub last_action: Option<String>,
1080    /// Set together with `last_action`; the pair moves as one, and both being
1081    /// absent means no escalation has ever been taken rather than that the last
1082    /// one succeeded.
1083    pub last_action_ms: Option<u64>,
1084}
1085
1086impl Default for ModuleHealthStatus {
1087    fn default() -> Self {
1088        Self {
1089            status: SupervisorHealthStatus::Unknown,
1090            last_probe_ms: None,
1091            detail: None,
1092            metrics: None,
1093            consecutive_failures: 0,
1094            late_answer_count: 0,
1095            last_late_answer_latency_ms: None,
1096            last_action: None,
1097            last_action_ms: None,
1098        }
1099    }
1100}
1101
1102/// Typed lifecycle state for a supervised module.
1103#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1104pub enum ModuleState {
1105    Starting,
1106    Running,
1107    Unresponsive,
1108    Restarting,
1109    Draining,
1110    Stopped,
1111    Failed,
1112    Disabled,
1113}
1114
1115impl fmt::Display for ModuleState {
1116    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1117        f.write_str(match self {
1118            Self::Starting => "starting",
1119            Self::Running => "running",
1120            Self::Unresponsive => "unresponsive",
1121            Self::Restarting => "restarting",
1122            Self::Draining => "draining",
1123            Self::Stopped => "stopped",
1124            Self::Failed => "failed",
1125            Self::Disabled => "disabled",
1126        })
1127    }
1128}
1129
1130/// Supervisor classification of a child-process exit.
1131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub enum ExitKind {
1133    Clean,
1134    Crash,
1135    DeliberateSeverance,
1136}
1137
1138impl From<ExitKind> for TerminalExitKind {
1139    fn from(kind: ExitKind) -> Self {
1140        match kind {
1141            ExitKind::Clean => Self::Clean,
1142            ExitKind::Crash => Self::Crash,
1143            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1144        }
1145    }
1146}
1147
1148/// Exact process identity retained when a supervised module registers its
1149/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1150#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1151pub(crate) struct ProcessIdentity {
1152    pub(crate) pid: u32,
1153    pub(crate) start_time: u64,
1154}
1155
1156/// Last observed child exit, if any.
1157#[derive(Debug, Clone, PartialEq, Eq)]
1158pub struct ExitReport {
1159    pub kind: ExitKind,
1160    pub code: Option<i32>,
1161    pub signal: Option<i32>,
1162    pub at_ms: u64,
1163}
1164
1165/// Point-in-time module status answerable by subc without forwarding to the
1166/// module process.
1167#[derive(Debug, Clone, PartialEq)]
1168pub struct ModuleStatus {
1169    pub module_id: String,
1170    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1171    pub waiting_for_previous_pid: Option<u32>,
1172    /// Why the previous instance still prevents a new start.
1173    pub waiting_for_previous_detail: Option<String>,
1174    /// The daemon's Windows per-spawn record: `pipe` after the process it
1175    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1176    /// List/status reports use this record, not the source the module sends
1177    /// in its registration message's manifest.provenance.launch_nonce_source.
1178    pub launch_nonce_source: Option<String>,
1179    pub state: ModuleState,
1180    pub enabled: bool,
1181    pub process_alive: bool,
1182    pub registration_active: bool,
1183    /// The module's declared wire protocol, carried beside `live` because it is
1184    /// what makes `live` readable: the two fields answer one question together.
1185    /// While a process is alive this is its launch declaration, not a later
1186    /// pending-reload edit. When down it is the configured next launch protocol.
1187    pub protocol: ModuleProtocol,
1188    /// Whether the module is serving, under the strongest definition the daemon
1189    /// can assert for its protocol.
1190    ///
1191    /// A subc module must also be REGISTERED: its process being alive says
1192    /// nothing about whether it can take a request. A `protocol: "none"` module
1193    /// never registers, so that term is dropped and this falls back to "enabled,
1194    /// running, and the process the daemon launched is alive" -- which is all
1195    /// the daemon observes about a process that speaks no subc wire. It stays a
1196    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1197    /// rather than printing it bare.
1198    pub live: bool,
1199    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1200    /// restarts have already released their slot, so this count can go down
1201    /// without anybody touching the module.
1202    pub restart_count: u32,
1203    /// Replacement processes spawned over this module's entire supervisor lifetime;
1204    /// unlike `restart_count`, this value is never reset by an operator action
1205    /// and never falls out of a window.
1206    pub lifetime_restarts: u32,
1207    pub spawn_generation: u64,
1208    /// The budget `restart_count` is spent against. Carried alongside the count
1209    /// because the count alone does not say how close the module is to being
1210    /// disabled, and reporting one without the other is what makes an
1211    /// about-to-be-retired module look ordinary.
1212    pub max_restarts: u32,
1213    /// The span `restart_count` is counted over. Carried with the pair above for
1214    /// the same reason they are carried together: "2 of 3" means one thing for a
1215    /// ten-minute window and something else entirely for a lifetime.
1216    pub restart_window: Duration,
1217    /// Effective drain and restart timing policy used by this running module.
1218    /// These values are carried together with the restart budget so status
1219    /// readers can compare configured intent with what the supervisor applied.
1220    pub drain_timeout: Duration,
1221    pub restart_backoff: Duration,
1222    pub restart_max_backoff: Duration,
1223    /// The module's process. On macOS this stays absent while the `ck-subc`
1224    /// launch trampoline is still running in that pid, and appears once the
1225    /// exec acknowledgement confirms the module image has replaced it. Launch
1226    /// time and the supervisor's own hold on the process are unaffected.
1227    pub pid: Option<u32>,
1228    pub spawned_at_ms: Option<u64>,
1229    pub spawned_from: Option<PathBuf>,
1230    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1231    pub process_start_time: Option<u64>,
1232    pub last_exit: Option<ExitReport>,
1233    pub health: ModuleHealthStatus,
1234}
1235
1236#[cfg(test)]
1237#[derive(Debug, Clone, Copy, PartialEq)]
1238struct ActorSelectCheckpoint {
1239    generation: u64,
1240    turn: u64,
1241    registered_connection: Option<ConnectionId>,
1242    next_probe_at: Option<Instant>,
1243    wake_after: Option<Duration>,
1244}
1245
1246#[derive(Debug, Clone, PartialEq)]
1247struct SupervisorSnapshot {
1248    subtree_target: crate::subtree_resources::Target,
1249    failed_retry: FailedRetry,
1250    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1251    #[cfg(windows)]
1252    pipe_consumed: Option<Arc<OnceLock<()>>>,
1253    /// Counts running-child loop turns, not executor polls of a parked wait.
1254    #[cfg(test)]
1255    actor_turns: u64,
1256    /// Running select after exec confirmation and with no unconsumed registry
1257    /// event. Recording a spawn or starting a loop turn is not this barrier.
1258    #[cfg(test)]
1259    actor_select: Option<ActorSelectCheckpoint>,
1260    state: ModuleState,
1261    enabled: bool,
1262    process_alive: bool,
1263    spawned_protocol: Option<ModuleProtocol>,
1264    spawn_failure: Option<String>,
1265    waiting_for_previous_pid: Option<u32>,
1266    waiting_for_previous_detail: Option<String>,
1267    /// When each crash restart was spent, oldest first. This IS the crash
1268    /// budget: its in-window length is the count an operator sees and the count
1269    /// the restart decision is made against, so there is no second counter that
1270    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1271    /// operator actions that used to zero the old lifetime counter.
1272    crash_restarts: VecDeque<Instant>,
1273    lifetime_restarts: u32,
1274    /// Successful child spawns in this daemon incarnation.
1275    ///
1276    /// `lifetime_restarts` was considered and rejected: it starts at zero
1277    /// (line 640), successful initial/operator spawns in `set_running` do not
1278    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1279    /// increments before a successful replacement exists (lines 604, 3846,
1280    /// and 3921), so a failed spawn can consume it. This counter moves only
1281    /// when a live PID is accepted below.
1282    spawn_generation: u64,
1283    pid: Option<u32>,
1284    #[cfg(target_os = "macos")]
1285    report_ready: Option<Arc<OnceLock<()>>>,
1286    /// Last reaped child, retained after current process facts are cleared.
1287    reaped_pid: Option<u32>,
1288    /// Whether a replacement has already been scheduled for the current process.
1289    respawn_pending: bool,
1290    /// A second restart is waiting for the replacement already scheduled.
1291    coalesced_restart_pending: bool,
1292    spawned_at_ms: Option<u64>,
1293    spawned_from: Option<PathBuf>,
1294    launch_spec_digest: Option<LaunchSpecDigest>,
1295    spawned_file_identity: Option<SpawnedFileIdentity>,
1296    process_start_time: Option<u64>,
1297    deliberate_severance: Option<ProcessIdentity>,
1298    last_exit: Option<ExitReport>,
1299    /// Diagnostic attached to the next drain's terminal record, if any.
1300    drain_disposition_detail: Option<String>,
1301    health: ModuleHealthStatus,
1302    /// Whether the current process was started as a swap candidate and so
1303    /// lives in the module's alternate cgroup. The next swap's candidate takes
1304    /// the other one, so the two processes of a swap never share a cgroup. A
1305    /// plain spawn always uses the primary cgroup.
1306    in_alternate_slot: bool,
1307    /// Whether the current `Draining` state ends in a replacement process
1308    /// (restart, reload, health restart) rather than a stop. Only meaningful
1309    /// while `state` is `Draining`; every entry into that state rewrites it.
1310    /// It is what lets route.open answer the retryable `module_reloading` to a
1311    /// consumer that reaches a still-registered process mid-restart, instead of
1312    /// the `supervisor_not_live` a stop or disable deserves.
1313    draining_to_replace: bool,
1314    /// Whether a configuration update has been applied since the current
1315    /// process was spawned, so that process runs an older spec than the one
1316    /// the supervisor now holds. A queued restart is only coalesced into a
1317    /// fresher process when this is false: a restart requested to pick up a
1318    /// new configuration must not be satisfied by a process that predates it.
1319    configuration_updated_since_spawn: bool,
1320}
1321
1322impl SupervisorSnapshot {
1323    /// The pid that status, provenance and resource readings may report. While
1324    /// the launch trampoline still runs in the pid, reading its executable or
1325    /// resource use would describe `ck-subc`, not the module, so none is
1326    /// reported until the exec acknowledgement confirms the module image.
1327    fn reported_pid(&self) -> Option<u32> {
1328        #[cfg(target_os = "macos")]
1329        if self
1330            .report_ready
1331            .as_ref()
1332            .is_some_and(|ready| ready.get().is_none())
1333        {
1334            return None;
1335        }
1336        self.pid
1337    }
1338
1339    fn starting() -> Self {
1340        Self::new(ModuleState::Starting, true)
1341    }
1342
1343    fn disabled() -> Self {
1344        Self::new(ModuleState::Disabled, false)
1345    }
1346
1347    /// Crash restarts still inside `window`, having dropped the ones that are
1348    /// not. Pruning on read is what makes the budget a rate: an instant older
1349    /// than the window stops holding a slot the moment anybody counts.
1350    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1351        while let Some(oldest) = self.crash_restarts.front() {
1352            if now.duration_since(*oldest) > window {
1353                self.crash_restarts.pop_front();
1354            } else {
1355                break;
1356            }
1357        }
1358        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1359    }
1360
1361    /// Spend one unit of the crash budget and record the restart in the ledger.
1362    ///
1363    /// The ring is bounded by the cap because more than `max_restarts` in-window
1364    /// instants can never be reached (the caller refuses the restart first), so
1365    /// anything beyond that is an unbounded queue waiting to happen.
1366    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1367        self.crash_restarts.push_back(now);
1368        while self.crash_restarts.len() > policy.max_restarts as usize {
1369            self.crash_restarts.pop_front();
1370        }
1371        self.lifetime_restarts += 1;
1372    }
1373
1374    /// Reserve one crash-restart slot and calculate the delay before respawning.
1375    /// The count is captured before recording this restart, so the first retry
1376    /// uses the base delay and each later in-window retry escalates once.
1377    fn next_crash_restart(
1378        &mut self,
1379        policy: &RestartPolicy,
1380        now: Instant,
1381    ) -> Option<CrashRestartSchedule> {
1382        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1383        if restart_in_window >= policy.max_restarts {
1384            return None;
1385        }
1386        self.record_crash_restart(policy, now);
1387        Some(CrashRestartSchedule {
1388            restart_in_window,
1389            delay: policy.delay_for_restart(restart_in_window),
1390        })
1391    }
1392
1393    /// Give the module its full budget back, as an operator restart, reload, or
1394    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1395    /// ledger of what actually happened, and an operator action does not unmake
1396    /// the crashes.
1397    fn clear_crash_restarts(&mut self) {
1398        self.crash_restarts.clear();
1399    }
1400
1401    fn new(state: ModuleState, enabled: bool) -> Self {
1402        Self {
1403            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1404            failed_retry: FailedRetry::default(),
1405            exec_outcome: None,
1406            #[cfg(test)]
1407            actor_turns: 0,
1408            #[cfg(test)]
1409            actor_select: None,
1410            state,
1411            enabled,
1412            process_alive: false,
1413            #[cfg(windows)]
1414            pipe_consumed: None,
1415            spawned_protocol: None,
1416            spawn_failure: None,
1417            waiting_for_previous_pid: None,
1418            waiting_for_previous_detail: None,
1419            crash_restarts: VecDeque::new(),
1420            lifetime_restarts: 0,
1421            spawn_generation: 0,
1422            pid: None,
1423            #[cfg(target_os = "macos")]
1424            report_ready: None,
1425            reaped_pid: None,
1426            respawn_pending: false,
1427            coalesced_restart_pending: false,
1428            spawned_at_ms: None,
1429            spawned_from: None,
1430            launch_spec_digest: None,
1431            spawned_file_identity: None,
1432            process_start_time: None,
1433            deliberate_severance: None,
1434            last_exit: None,
1435            drain_disposition_detail: None,
1436            health: ModuleHealthStatus::default(),
1437            in_alternate_slot: false,
1438            draining_to_replace: false,
1439            configuration_updated_since_spawn: false,
1440        }
1441    }
1442}
1443
1444type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1445
1446type SpawnSubscriberKey = (ConnectionId, u64);
1447
1448#[derive(Debug)]
1449struct SpawnSubscriber {
1450    version: u8,
1451    frames: mpsc::Sender<Frame>,
1452    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1453    /// from which event. The full frame channel cannot carry that news, so it
1454    /// travels beside it; see `SpawnEventFeed::subscribe`.
1455    lagged: Option<oneshot::Sender<SpawnCursor>>,
1456}
1457
1458#[derive(Debug)]
1459struct SpawnEventState {
1460    daemon_incarnation: String,
1461    seq: u64,
1462    capacity: usize,
1463    live: HashMap<String, LiveSpawn>,
1464    generations: HashMap<String, u64>,
1465    events: VecDeque<SpawnEvent>,
1466    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1467}
1468
1469impl Default for SpawnEventState {
1470    fn default() -> Self {
1471        Self {
1472            daemon_incarnation: "unconfigured".to_string(),
1473            seq: 0,
1474            capacity: SPAWN_EVENT_RING_CAPACITY,
1475            live: HashMap::new(),
1476            generations: HashMap::new(),
1477            events: VecDeque::new(),
1478            subscribers: HashMap::new(),
1479        }
1480    }
1481}
1482
1483#[derive(Debug, Clone, Default)]
1484struct SpawnEventFeed(
1485    Arc<Mutex<SpawnEventState>>,
1486    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1487);
1488
1489#[derive(Debug, Clone, PartialEq, Eq)]
1490pub(crate) enum SpawnSubscribeRefusal {
1491    ForeignIncarnation { current: String },
1492    TooOld { oldest: SpawnCursor },
1493    Frame(String),
1494}
1495
1496impl SpawnEventFeed {
1497    fn configure_incarnation(&self, daemon_incarnation: String) {
1498        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1499        state.daemon_incarnation = daemon_incarnation;
1500        state.seq = 0;
1501        state.live.clear();
1502        state.generations.clear();
1503        state.events.clear();
1504        state.subscribers.clear();
1505    }
1506
1507    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1508        SpawnCursor {
1509            daemon_incarnation: state.daemon_incarnation.clone(),
1510            seq: state.seq,
1511        }
1512    }
1513
1514    fn snapshot(&self) -> SpawnSnapshot {
1515        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1516        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1517        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1518        SpawnSnapshot {
1519            cursor: Self::cursor(&state),
1520            ring_bound: state.capacity as u64,
1521            live,
1522        }
1523    }
1524
1525    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1526        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1527        let generation = state
1528            .generations
1529            .get(module_id)
1530            .copied()
1531            .unwrap_or(0)
1532            .checked_add(1)
1533            .expect("spawn generation exhausted");
1534        state.generations.insert(module_id.to_string(), generation);
1535        let live = LiveSpawn {
1536            module_id: module_id.to_string(),
1537            spawn_generation: generation,
1538            pid,
1539            spawned_at_ms,
1540        };
1541        state.live.insert(module_id.to_string(), live);
1542        Self::emit_locked(
1543            &mut state,
1544            SpawnEventKind::Spawned,
1545            module_id.to_string(),
1546            generation,
1547            pid,
1548            None,
1549            None,
1550        );
1551        generation
1552    }
1553
1554    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1555        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1556        let Some(live) = state.live.remove(module_id) else {
1557            warn!(
1558                module_id,
1559                "terminal record had no live spawn event identity"
1560            );
1561            return;
1562        };
1563        Self::emit_locked(
1564            &mut state,
1565            SpawnEventKind::Exited,
1566            module_id.to_string(),
1567            live.spawn_generation,
1568            live.pid,
1569            exit_code,
1570            exit_signal,
1571        );
1572    }
1573
1574    /// Report the exit of a process that a swap has already replaced.
1575    ///
1576    /// `emit_exited` removes the module's live entry, which after a swap's
1577    /// cutover describes the promoted candidate, not the old process now
1578    /// exiting. This emits the old generation's exit and leaves the live entry
1579    /// alone unless it still names that generation.
1580    fn emit_superseded_exited(
1581        &self,
1582        module_id: &str,
1583        spawn_generation: u64,
1584        pid: u32,
1585        exit_code: Option<i32>,
1586        exit_signal: Option<i32>,
1587    ) {
1588        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1589        if state
1590            .live
1591            .get(module_id)
1592            .is_some_and(|live| live.spawn_generation == spawn_generation)
1593        {
1594            state.live.remove(module_id);
1595        }
1596        Self::emit_locked(
1597            &mut state,
1598            SpawnEventKind::Exited,
1599            module_id.to_string(),
1600            spawn_generation,
1601            pid,
1602            exit_code,
1603            exit_signal,
1604        );
1605    }
1606
1607    #[allow(clippy::too_many_arguments)]
1608    fn emit_locked(
1609        state: &mut SpawnEventState,
1610        kind: SpawnEventKind,
1611        module_id: String,
1612        spawn_generation: u64,
1613        pid: u32,
1614        exit_code: Option<i32>,
1615        exit_signal: Option<i32>,
1616    ) {
1617        state.seq = state
1618            .seq
1619            .checked_add(1)
1620            .expect("spawn event sequence exhausted");
1621        let event = SpawnEvent {
1622            cursor: Self::cursor(state),
1623            kind,
1624            module_id,
1625            spawn_generation,
1626            pid,
1627            exit_code,
1628            exit_signal,
1629        };
1630        state.events.push_back(event.clone());
1631        while state.events.len() > state.capacity {
1632            state.events.pop_front();
1633        }
1634        let body = match serde_json::to_vec(&event) {
1635            Ok(body) => body,
1636            Err(error) => {
1637                error!(%error, "failed to serialize supervisor spawn event");
1638                return;
1639            }
1640        };
1641        state.subscribers.retain(|(connection_id, corr), subscriber| {
1642            let frame = Frame::build_with_version(
1643                subscriber.version,
1644                FrameType::StreamData,
1645                control_flags(),
1646                0,
1647                0,
1648                *corr,
1649                body.clone(),
1650            );
1651            match frame {
1652                Ok(frame) => {
1653                    if subscriber.frames.try_send(frame).is_ok() {
1654                        true
1655                    } else {
1656                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1657                        if let Some(lagged) = subscriber.lagged.take() {
1658                            let _ = lagged.send(event.cursor.clone());
1659                        }
1660                        false
1661                    }
1662                }
1663                Err(error) => {
1664                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1665                    false
1666                }
1667            }
1668        });
1669    }
1670
1671    fn subscribe(
1672        &self,
1673        connection_id: ConnectionId,
1674        corr: u64,
1675        version: u8,
1676        since: Option<SpawnCursor>,
1677        sink: FrameSink,
1678    ) -> Result<(), SpawnSubscribeRefusal> {
1679        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1680        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1681        {
1682            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1683            let replay = if let Some(since) = since {
1684                if since.daemon_incarnation != state.daemon_incarnation {
1685                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1686                        current: state.daemon_incarnation.clone(),
1687                    });
1688                }
1689                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1690                    if since.seq < oldest.seq.saturating_sub(1) {
1691                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1692                    }
1693                }
1694                state
1695                    .events
1696                    .iter()
1697                    .filter(|event| event.cursor.seq > since.seq)
1698                    .cloned()
1699                    .collect::<Vec<_>>()
1700            } else {
1701                Vec::new()
1702            };
1703            for event in replay {
1704                let body = serde_json::to_vec(&event)
1705                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1706                let frame = Frame::build_with_version(
1707                    version,
1708                    FrameType::StreamData,
1709                    control_flags(),
1710                    0,
1711                    0,
1712                    corr,
1713                    body,
1714                )
1715                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716                frames
1717                    .try_send(frame)
1718                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1719            }
1720            state.subscribers.insert(
1721                (connection_id, corr),
1722                SpawnSubscriber {
1723                    version,
1724                    frames,
1725                    lagged: Some(lagged),
1726                },
1727            );
1728        }
1729        // The lagged terminal is sent here, by the forwarder, rather than by
1730        // the emitter: at the moment of the drop the subscriber's own channel
1731        // is full, and writing to the connection sink directly from the emitter
1732        // would put the Error AHEAD of the events still queued in that channel
1733        // (and the emitter holds the feed lock, so it cannot await the sink).
1734        // Dropping the subscriber drops the only sender, so `recv` drains every
1735        // queued event and then returns `None`; only then is the Error sent, so
1736        // the client sees each event it can keep, then the reason it was cut.
1737        // Cancel and connection removal drop the oneshot unsent, so they end
1738        // the stream with no Error.
1739        tokio::spawn(async move {
1740            while let Some(frame) = receiver.recv().await {
1741                if sink.send(frame).await.is_err() {
1742                    return;
1743                }
1744            }
1745            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1746                return;
1747            };
1748            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1749                Ok(frame) => {
1750                    let _ = sink.send(frame).await;
1751                }
1752                Err(error) => {
1753                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1754                }
1755            }
1756        });
1757        Ok(())
1758    }
1759
1760    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1761        let Some(subscriber) = self
1762            .0
1763            .lock()
1764            .unwrap_or_else(|p| p.into_inner())
1765            .subscribers
1766            .remove(&(connection_id, corr))
1767        else {
1768            return false;
1769        };
1770        if let Ok(frame) = Frame::build_with_version(
1771            subscriber.version,
1772            FrameType::StreamEnd,
1773            control_flags(),
1774            0,
1775            0,
1776            corr,
1777            Vec::new(),
1778        ) {
1779            tokio::spawn(async move {
1780                let _ = subscriber.frames.send(frame).await;
1781            });
1782        }
1783        true
1784    }
1785
1786    fn remove_connection(&self, connection_id: ConnectionId) {
1787        self.0
1788            .lock()
1789            .unwrap_or_else(|p| p.into_inner())
1790            .subscribers
1791            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1792    }
1793
1794    #[cfg(any(test, feature = "test-support"))]
1795    fn set_capacity(&self, capacity: usize) {
1796        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1797    }
1798
1799    #[cfg(any(test, feature = "test-support"))]
1800    fn subscriber_count(&self) -> usize {
1801        self.0
1802            .lock()
1803            .unwrap_or_else(|p| p.into_inner())
1804            .subscribers
1805            .len()
1806    }
1807}
1808
1809/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1810/// The terminal Error a lagged spawn subscriber receives after its queued events.
1811fn spawn_subscriber_lagged_frame(
1812    version: u8,
1813    corr: u64,
1814    first_undelivered: SpawnCursor,
1815) -> Result<Frame, String> {
1816    // The daemon authors this terminal Error, so it carries the daemon-origin bit.
1817    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1818        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1819        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1820            .to_string(),
1821        detail: Some(serde_json::json!({
1822            "first_undelivered_cursor": first_undelivered
1823        })),
1824    })
1825    .map_err(|error| error.to_string())?;
1826    Frame::build_with_version(
1827        version,
1828        FrameType::Error,
1829        crate::router::daemon_error_flags(),
1830        0,
1831        0,
1832        corr,
1833        body,
1834    )
1835    .map_err(|error| error.to_string())
1836}
1837
1838pub trait ModuleProcessLiveness: Send + Sync {
1839    fn process_live(&self, module_id: &str) -> Option<bool>;
1840
1841    /// Whether the supervisor is replacing this module's process right now: an
1842    /// operator restart or reload, a health restart, or a crash respawn whose
1843    /// backoff is running. A module in that state is not live, but a consumer
1844    /// refused now should retry shortly rather than treat the target as gone.
1845    /// Stopped, failed, and disabled modules are not replacing.
1846    fn process_replacing(&self, _module_id: &str) -> bool {
1847        false
1848    }
1849}
1850
1851/// Shared process-liveness registry keyed by supervised `module_id`.
1852#[derive(Debug, Clone, Default)]
1853pub struct SupervisorProcessLiveness {
1854    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1855}
1856
1857impl SupervisorProcessLiveness {
1858    pub fn new() -> Self {
1859        Self::default()
1860    }
1861
1862    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1863        let mut snapshots = self
1864            .snapshots
1865            .lock()
1866            .unwrap_or_else(|poisoned| poisoned.into_inner());
1867        snapshots.insert(module_id, snapshot);
1868    }
1869
1870    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1871        let mut snapshots = self
1872            .snapshots
1873            .lock()
1874            .unwrap_or_else(|poisoned| poisoned.into_inner());
1875        let is_current = snapshots
1876            .get(module_id)
1877            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1878            .unwrap_or(false);
1879        if is_current {
1880            snapshots.remove(module_id);
1881        }
1882    }
1883}
1884
1885impl ModuleProcessLiveness for SupervisorProcessLiveness {
1886    fn process_live(&self, module_id: &str) -> Option<bool> {
1887        let snapshot = {
1888            let snapshots = self
1889                .snapshots
1890                .lock()
1891                .unwrap_or_else(|poisoned| poisoned.into_inner());
1892            snapshots.get(module_id).cloned()
1893        }?;
1894        let snapshot = snapshot
1895            .lock()
1896            .unwrap_or_else(|poisoned| poisoned.into_inner());
1897        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1898    }
1899
1900    fn process_replacing(&self, module_id: &str) -> bool {
1901        let Some(snapshot) = self
1902            .snapshots
1903            .lock()
1904            .unwrap_or_else(|poisoned| poisoned.into_inner())
1905            .get(module_id)
1906            .cloned()
1907        else {
1908            return false;
1909        };
1910        let snapshot = snapshot
1911            .lock()
1912            .unwrap_or_else(|poisoned| poisoned.into_inner());
1913        snapshot.enabled
1914            && match snapshot.state {
1915                ModuleState::Restarting => true,
1916                ModuleState::Draining => snapshot.draining_to_replace,
1917                ModuleState::Starting
1918                | ModuleState::Running
1919                | ModuleState::Unresponsive
1920                | ModuleState::Stopped
1921                | ModuleState::Failed
1922                | ModuleState::Disabled => false,
1923            }
1924    }
1925}
1926
1927#[cfg(test)]
1928#[derive(Debug, Default)]
1929struct ReloadExitRecordGate {
1930    reached: tokio::sync::Notify,
1931    resume: tokio::sync::Notify,
1932}
1933
1934#[derive(Debug, Clone, Copy)]
1935enum RespawnKind {
1936    Spawn,
1937    Reload,
1938}
1939
1940#[derive(Debug, Clone, Copy)]
1941struct PendingRespawn {
1942    deadline: Instant,
1943    kind: RespawnKind,
1944    daemon_initiated: bool,
1945}
1946
1947type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1948
1949#[derive(Debug, Clone)]
1950struct SupervisorRuntimeConfig {
1951    registry: Arc<Registry>,
1952    /// Retained across an operation's deferred backoff, until its child starts.
1953    recovery_is_daemon: bool,
1954    start_requester: Option<Principal>,
1955    pending_activity_restart: bool,
1956    provenance_probe: ExecutableIdentityProbe,
1957    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1958    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1959    /// A reload acknowledges completion only after its replacement registers.
1960    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1961    restart_policy: RestartPolicy,
1962    register_timeout: Duration,
1963    /// This module's RESOLVED drain budget: per-module config when present,
1964    /// else `default_drain_timeout`.
1965    drain_timeout: Duration,
1966    /// Shared with the status handle so the attested value changes atomically
1967    /// when a rescan updates the running drain policy.
1968    effective_drain_timeout: Arc<Mutex<Duration>>,
1969    /// The supervisor-wide fallback, kept so a configuration update that
1970    /// REMOVES the per-module override can re-resolve to it.
1971    default_drain_timeout: Duration,
1972    health: HealthConfig,
1973    connection_file_path: Option<PathBuf>,
1974    capture_logs_dir: Option<PathBuf>,
1975    forwarding: Option<Arc<ForwardingTable>>,
1976    /// The shared handle, so every spawn path (initial, restart, reload) records the
1977    /// reserved-module launch nonce the HELLO verifier checks against.
1978    supervisor_handle: Option<SupervisorHandle>,
1979    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1980    /// status queries.
1981    ///
1982    /// One ring per module, held across every respawn. The lines explaining an exit
1983    /// are written BEFORE that exit, so a ring recreated per process would be empty
1984    /// exactly when it is asked for.
1985    stderr_ring: Arc<Mutex<StderrRing>>,
1986    terminal_ring: Arc<Mutex<TerminalRing>>,
1987    spawn_events: SpawnEventFeed,
1988    child_roster: ChildRoster,
1989    #[cfg(target_os = "linux")]
1990    cgroup_placement: Option<subc_cgroup::Placement>,
1991    #[cfg(test)]
1992    test_seed_stale_facts_before_enable_spawn: bool,
1993    #[cfg(test)]
1994    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1995}
1996
1997impl SupervisorRuntimeConfig {
1998    fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1999        FailureContext {
2000            daemon_initiated,
2001            shutdown: self.child_roster.shutdown_flag(),
2002        }
2003    }
2004}
2005
2006#[derive(Debug, Clone, PartialEq, Eq)]
2007struct SupervisedConfiguration {
2008    spec: ModuleSpec,
2009    health: HealthConfig,
2010}
2011
2012/// Shared daemon lookup table for supervised module handles.
2013///
2014/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
2015/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
2016/// launch nonces recorded at spawn are checked by the same daemon instance.
2017#[derive(Debug, Clone, Default)]
2018pub struct SupervisorHandle {
2019    restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2020    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2021    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2022    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2023    /// Module ids the supervisor has taken on. An id is added BEFORE the
2024    /// module's first process is spawned and removed only when the module
2025    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
2026    /// the keys of `modules`.
2027    ///
2028    /// `modules` cannot answer "is this module configured?" on its own: a
2029    /// [`SupervisedModule`] only exists once its process has been spawned, and
2030    /// a fast child can connect, register, sync its scopes and ask about them
2031    /// before the supervisor has inserted it. Answering "not configured" in that
2032    /// gap makes scope admission refuse with the terminal "will never sync"
2033    /// instead of the retryable "has not synced yet".
2034    configured_ids: Arc<Mutex<HashSet<String>>>,
2035    spawn_events: SpawnEventFeed,
2036    /// The current expected launch nonce for each reserved module_id. Set when the
2037    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
2038    /// non-reserved module never has an entry here and is never nonce-checked.
2039    /// Reserved module ids and the nonce that authorizes their next HELLO.
2040    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
2041    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
2042    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
2043    /// had NO entry and admitted anyone: the reservation protected the nonce
2044    /// holder, not the NAME (found live by CKCRED's canary probe registering
2045    /// against a reserved scratch id).
2046    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2047    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2048    ///
2049    /// This is deliberately in-memory only: subc is state-free across daemon
2050    /// restarts, and the tombstone only explains the hours-after-removal window
2051    /// while this executing daemon is still alive. Do not persist it in a store.
2052    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2053    /// The current launch nonce for every supervised spawn. This is separate from
2054    /// reserved_nonces because consumer route.open attestation applies to all spawned
2055    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2056    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2057    /// Reserved namespace prefixes mapped to the supervised owner module whose
2058    /// current spawn nonce authorizes HELLO claims below the prefix.
2059    ///
2060    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2061    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2062    /// accidental collisions and lower-trust processes from squatting protected
2063    /// namespaces.
2064    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2065    /// Blue/green swaps in progress, by module id. An entry exists from just
2066    /// before the candidate process is spawned until the swap has failed, or
2067    /// has cut over and the old process is gone. While it exists, HELLO for the
2068    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2069    /// consumer attestation accepts both processes' nonces.
2070    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2071    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2072    promotion_observer: PromotionObserverSlot,
2073    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2074    /// this daemon-wide ordering, a rescan could retire or update a module while a
2075    /// concurrent reload still held its old handle and launch specification.
2076    operation_lock: Arc<AsyncMutex<()>>,
2077}
2078
2079/// Told when a swap has promoted its candidate to be the module's active
2080/// registration.
2081///
2082/// An ordinary HELLO runs the control plane's registration side effects (the
2083/// capability cache, the deny census, the requirement recompute) as it
2084/// registers. A swap candidate's HELLO does not, because it is not routable;
2085/// promotion is when those must run instead, and promotion happens in the
2086/// supervisor, which has no other way into the control handler.
2087pub(crate) trait SwapPromotionObserver: Send + Sync {
2088    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2089}
2090
2091/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2092/// control handler) owns this handle, so a strong reference back would be a
2093/// cycle that keeps both alive.
2094#[derive(Clone, Default)]
2095struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2096
2097impl fmt::Debug for PromotionObserverSlot {
2098    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2099        f.write_str("PromotionObserverSlot")
2100    }
2101}
2102
2103/// The nonces of one open swap.
2104#[derive(Debug, Clone)]
2105struct OpenSwap {
2106    /// The launch nonce minted for the candidate process. It is the swap
2107    /// token: the only thing that admits a HELLO into the candidate slot.
2108    candidate_nonce: String,
2109    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2110    /// here because cutover moves the module's recorded spawn nonce to the
2111    /// candidate while the incumbent is still draining and its consumers are
2112    /// still attesting with this one.
2113    incumbent_nonce: Option<String>,
2114    /// Set once a HELLO has been admitted with the swap token, so the token
2115    /// admits one registration and cannot be replayed after cutover empties
2116    /// the candidate slot.
2117    candidate_admitted: bool,
2118}
2119
2120/// What the swap gate says about a HELLO. See
2121/// [`SupervisorHandle::swap_hello_admission`].
2122#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2123pub(crate) enum SwapHelloAdmission {
2124    /// No swap is open for the id (or the HELLO carries the incumbent's own
2125    /// nonce); the ordinary gates decide.
2126    NotSwapping,
2127    /// The HELLO carries the swap token: register it into the candidate slot.
2128    Candidate,
2129    /// A swap is open and the HELLO carries a nonce the supervisor did not
2130    /// mint for this id, no nonce, or a token already used.
2131    Refused,
2132}
2133
2134#[derive(Debug, Clone, PartialEq, Eq)]
2135pub(crate) enum ReservedHelloRejection {
2136    Exact {
2137        module_id: String,
2138    },
2139    Prefix {
2140        prefix: String,
2141        owner_module_id: String,
2142    },
2143}
2144
2145#[derive(Debug)]
2146struct DaemonRestartIntent {
2147    expires: Option<Duration>,
2148    window: Duration,
2149}
2150
2151impl Default for DaemonRestartIntent {
2152    fn default() -> Self {
2153        Self {
2154            expires: None,
2155            window: Duration::from_secs(120),
2156        }
2157    }
2158}
2159
2160impl SupervisorHandle {
2161    pub(crate) fn announce_daemon_restart(&self) -> u64 {
2162        let mut intent = self
2163            .restart_intent
2164            .lock()
2165            .unwrap_or_else(|p| p.into_inner());
2166        intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2167        unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2168    }
2169
2170    #[cfg(feature = "test-support")]
2171    pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2172        self.restart_intent
2173            .lock()
2174            .unwrap_or_else(|p| p.into_inner())
2175            .window = window;
2176    }
2177
2178    #[cfg(unix)]
2179    fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2180        let expiry = self
2181            .restart_intent
2182            .lock()
2183            .unwrap_or_else(|p| p.into_inner())
2184            .expires
2185            .take();
2186        if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2187            RouteCloseReason::Restart
2188        } else {
2189            RouteCloseReason::Disable
2190        }
2191    }
2192
2193    pub fn new() -> Self {
2194        Self::default()
2195    }
2196
2197    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2198        self.spawn_events.snapshot()
2199    }
2200
2201    pub(crate) fn subscribe_spawns(
2202        &self,
2203        connection_id: ConnectionId,
2204        corr: u64,
2205        version: u8,
2206        since: Option<SpawnCursor>,
2207        sink: FrameSink,
2208    ) -> Result<(), SpawnSubscribeRefusal> {
2209        self.spawn_events
2210            .subscribe(connection_id, corr, version, since, sink)
2211    }
2212
2213    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2214        self.spawn_events.cancel(connection_id, corr)
2215    }
2216
2217    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2218        self.spawn_events.remove_connection(connection_id);
2219    }
2220
2221    #[cfg(any(test, feature = "test-support"))]
2222    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2223        assert!(capacity > 0, "spawn event capacity must be non-zero");
2224        self.spawn_events.set_capacity(capacity);
2225    }
2226
2227    #[cfg(any(test, feature = "test-support"))]
2228    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2229        self.spawn_events.subscriber_count()
2230    }
2231
2232    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2233    /// a respawn invalidates stale consumer identities.
2234    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2235        self.spawn_nonces
2236            .lock()
2237            .unwrap_or_else(|poisoned| poisoned.into_inner())
2238            .insert(module_id.to_string(), nonce);
2239    }
2240
2241    /// Record the launch nonce expected from the next HELLO for a reserved module,
2242    /// replacing any prior nonce (a respawn invalidates the previous one).
2243    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2244        self.reserved_nonces
2245            .lock()
2246            .unwrap_or_else(|poisoned| poisoned.into_inner())
2247            .insert(module_id.to_string(), Some(nonce));
2248    }
2249
2250    /// Record namespace prefixes owned by a supervised module.
2251    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2252        let mut owners = self
2253            .reserved_prefix_owners
2254            .lock()
2255            .unwrap_or_else(|poisoned| poisoned.into_inner());
2256        owners.retain(|_, owner| owner != owner_module_id);
2257        for prefix in prefixes {
2258            owners.insert(prefix.clone(), owner_module_id.to_string());
2259        }
2260    }
2261
2262    /// The launch nonce most recently minted for a module's spawn, if any.
2263    #[cfg(test)]
2264    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2265        self.spawn_nonces
2266            .lock()
2267            .unwrap_or_else(|poisoned| poisoned.into_inner())
2268            .get(module_id)
2269            .cloned()
2270    }
2271
2272    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2273        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2274        let spawn_nonce = self
2275            .spawn_nonces
2276            .lock()
2277            .unwrap_or_else(|poisoned| poisoned.into_inner())
2278            .get(&spec.module_id)
2279            .cloned();
2280        let mut reserved_nonces = self
2281            .reserved_nonces
2282            .lock()
2283            .unwrap_or_else(|poisoned| poisoned.into_inner());
2284        if spec.reserved {
2285            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2286            // reserved name whose module has never spawned has no legitimate
2287            // holder, and the entry's absence is what used to leave the name
2288            // open to the first claimant.
2289            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2290        }
2291        drop(reserved_nonces);
2292        // A later unreserved declaration must not silently unreserve an id that
2293        // was retained after its reserved configuration was removed. The explicit
2294        // release ceremony is the only operation that retires that gate.
2295        self.removal_tombstones
2296            .lock()
2297            .unwrap_or_else(|poisoned| poisoned.into_inner())
2298            .remove(&spec.module_id);
2299    }
2300
2301    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2302    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2303    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2304    /// with no matching prefix are always authorized.
2305    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2306        self.reserved_hello_verification(module_id, presented)
2307            .is_ok()
2308    }
2309
2310    /// Whether HELLO passed a nonce check that was actually required by reserved
2311    /// module policy, or the refusal explaining why a required check failed.
2312    pub(crate) fn reserved_hello_verification(
2313        &self,
2314        module_id: &str,
2315        presented: Option<&str>,
2316    ) -> Result<bool, ReservedHelloRejection> {
2317        let nonces = self
2318            .reserved_nonces
2319            .lock()
2320            .unwrap_or_else(|poisoned| poisoned.into_inner());
2321        if let Some(expected) = nonces.get(module_id) {
2322            // `None` = reserved with no legitimate holder: refuse every
2323            // presentation, because no process can hold a nonce that was never
2324            // minted. Only a real minted nonce admits, in constant time.
2325            let authorized = match expected {
2326                Some(expected) => {
2327                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2328                }
2329                None => false,
2330            };
2331            if authorized {
2332                return Ok(true);
2333            }
2334            return Err(ReservedHelloRejection::Exact {
2335                module_id: module_id.to_string(),
2336            });
2337        }
2338        drop(nonces);
2339
2340        let matched_prefix = self
2341            .reserved_prefix_owners
2342            .lock()
2343            .unwrap_or_else(|poisoned| poisoned.into_inner())
2344            .iter()
2345            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2346            .max_by_key(|(prefix, _)| prefix.len())
2347            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2348        let Some((prefix, owner_module_id)) = matched_prefix else {
2349            return Ok(false);
2350        };
2351
2352        let authorized = presented.is_some_and(|presented| {
2353            self.spawn_nonces
2354                .lock()
2355                .unwrap_or_else(|poisoned| poisoned.into_inner())
2356                .get(&owner_module_id)
2357                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2358                // While the owner is being swapped, children started by
2359                // either of its two processes hold that process's nonce.
2360                || self.swap_nonce_matches(&owner_module_id, presented)
2361        });
2362        if authorized {
2363            Ok(true)
2364        } else {
2365            Err(ReservedHelloRejection::Prefix {
2366                prefix,
2367                owner_module_id,
2368            })
2369        }
2370    }
2371
2372    /// Whether a consumer connection proved it came from a daemon-spawned module.
2373    ///
2374    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2375    /// accepted only for module ids the supervisor has spawned.
2376    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2377        if presented.is_empty() {
2378            return false;
2379        }
2380        let nonces = self
2381            .spawn_nonces
2382            .lock()
2383            .unwrap_or_else(|poisoned| poisoned.into_inner());
2384        let current = nonces
2385            .get(module_id)
2386            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2387        drop(nonces);
2388        // During a swap two processes of the module are alive, and a consumer
2389        // started by either one presents that process's nonce. Accepting only
2390        // the recorded one would fail the incumbent's consumers for the whole
2391        // overlap once cutover moves the record to the candidate.
2392        current || self.swap_nonce_matches(module_id, presented)
2393    }
2394
2395    /// Whether `presented` is either nonce of an open swap for `module_id`.
2396    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2397        let swaps = self
2398            .swaps
2399            .lock()
2400            .unwrap_or_else(|poisoned| poisoned.into_inner());
2401        swaps.get(module_id).is_some_and(|swap| {
2402            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2403                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2404                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2405                })
2406        })
2407    }
2408
2409    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2410    /// Called before the candidate process exists.
2411    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2412        let incumbent_nonce = self
2413            .spawn_nonces
2414            .lock()
2415            .unwrap_or_else(|poisoned| poisoned.into_inner())
2416            .get(module_id)
2417            .cloned();
2418        self.swaps
2419            .lock()
2420            .unwrap_or_else(|poisoned| poisoned.into_inner())
2421            .insert(
2422                module_id.to_string(),
2423                OpenSwap {
2424                    candidate_nonce,
2425                    incumbent_nonce,
2426                    candidate_admitted: false,
2427                },
2428            );
2429    }
2430
2431    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2432    /// the module's recorded one.
2433    pub(crate) fn close_swap(&self, module_id: &str) {
2434        self.swaps
2435            .lock()
2436            .unwrap_or_else(|poisoned| poisoned.into_inner())
2437            .remove(module_id);
2438    }
2439
2440    /// Install the observer told about swap promotions, replacing any earlier
2441    /// one.
2442    pub(crate) fn set_swap_promotion_observer(
2443        &self,
2444        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2445    ) {
2446        *self
2447            .promotion_observer
2448            .0
2449            .lock()
2450            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2451    }
2452
2453    /// Tell the installed observer, if it is still alive, that a swap promoted
2454    /// `registration`.
2455    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2456        let observer = self
2457            .promotion_observer
2458            .0
2459            .lock()
2460            .unwrap_or_else(|poisoned| poisoned.into_inner())
2461            .as_ref()
2462            .and_then(std::sync::Weak::upgrade);
2463        if let Some(observer) = observer {
2464            observer.swap_promoted(registration);
2465        }
2466    }
2467
2468    /// Whether a swap is open for `module_id`.
2469    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2470        self.swaps
2471            .lock()
2472            .unwrap_or_else(|poisoned| poisoned.into_inner())
2473            .contains_key(module_id)
2474    }
2475
2476    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2477    /// respawn would, once cutover has made the candidate the module's process.
2478    /// The swap stays open so the incumbent's nonce keeps attesting until the
2479    /// incumbent has drained and exited.
2480    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2481        let candidate_nonce = self
2482            .swaps
2483            .lock()
2484            .unwrap_or_else(|poisoned| poisoned.into_inner())
2485            .get(module_id)
2486            .map(|swap| swap.candidate_nonce.clone());
2487        let Some(nonce) = candidate_nonce else {
2488            return;
2489        };
2490        self.set_spawn_nonce(module_id, nonce.clone());
2491        if reserved {
2492            self.set_reserved_nonce(module_id, nonce);
2493        }
2494    }
2495
2496    /// The swap gate for a HELLO claiming `module_id`.
2497    ///
2498    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2499    /// presents the candidate nonce, which the reserved gate (holding the
2500    /// incumbent's nonce) would refuse as `reserved_module` before swap
2501    /// admission was ever reached. And it applies to unreserved ids too: for an
2502    /// unreserved id the only thing that ever stopped a second process claiming
2503    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2504    /// refusal a swap lifts for its candidate.
2505    ///
2506    /// The incumbent's own nonce falls through to the ordinary gates, which
2507    /// treat it as they always have (a live incumbent is refused as a
2508    /// duplicate). Anything else while a swap is open is refused, including an
2509    /// absent nonce.
2510    pub(crate) fn swap_hello_admission(
2511        &self,
2512        module_id: &str,
2513        presented: Option<&str>,
2514    ) -> SwapHelloAdmission {
2515        let swaps = self
2516            .swaps
2517            .lock()
2518            .unwrap_or_else(|poisoned| poisoned.into_inner());
2519        let Some(swap) = swaps.get(module_id) else {
2520            return SwapHelloAdmission::NotSwapping;
2521        };
2522        let Some(presented) = presented else {
2523            return SwapHelloAdmission::Refused;
2524        };
2525        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2526            return if swap.candidate_admitted {
2527                SwapHelloAdmission::Refused
2528            } else {
2529                SwapHelloAdmission::Candidate
2530            };
2531        }
2532        if swap
2533            .incumbent_nonce
2534            .as_deref()
2535            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2536        {
2537            return SwapHelloAdmission::NotSwapping;
2538        }
2539        SwapHelloAdmission::Refused
2540    }
2541
2542    /// Record that the swap token has registered a candidate, so it admits no
2543    /// second HELLO.
2544    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2545        if let Some(swap) = self
2546            .swaps
2547            .lock()
2548            .unwrap_or_else(|poisoned| poisoned.into_inner())
2549            .get_mut(module_id)
2550        {
2551            swap.candidate_admitted = true;
2552        }
2553    }
2554
2555    /// Test/support lookup for the current launch nonce of a supervised spawn.
2556    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2557        self.spawn_nonces
2558            .lock()
2559            .unwrap_or_else(|poisoned| poisoned.into_inner())
2560            .get(module_id)
2561            .cloned()
2562    }
2563
2564    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2565    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2566        self.reserved_nonces
2567            .lock()
2568            .unwrap_or_else(|poisoned| poisoned.into_inner())
2569            .get(module_id)
2570            .cloned()
2571            .flatten()
2572    }
2573
2574    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2575        // Normally already marked before the process was spawned; marking here
2576        // too keeps `configured_ids` a superset of the roster for any caller
2577        // that inserts a module directly.
2578        self.mark_configured(module.module_id());
2579        let mut modules = self
2580            .modules
2581            .lock()
2582            .unwrap_or_else(|poisoned| poisoned.into_inner());
2583        modules.insert(module.module_id().to_string(), module)
2584    }
2585
2586    /// Record that the supervisor has taken on `module_id`. Called before the
2587    /// module's first process is spawned, so that by the time that process can
2588    /// register, [`Self::is_configured`] already answers true.
2589    fn mark_configured(&self, module_id: &str) {
2590        self.configured_ids
2591            .lock()
2592            .unwrap_or_else(|poisoned| poisoned.into_inner())
2593            .insert(module_id.to_string());
2594    }
2595
2596    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2597    /// before it was ever put on the roster. A module already on the roster
2598    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2599    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2600        let modules = self
2601            .modules
2602            .lock()
2603            .unwrap_or_else(|poisoned| poisoned.into_inner());
2604        if !modules.contains_key(module_id) {
2605            self.configured_ids
2606                .lock()
2607                .unwrap_or_else(|poisoned| poisoned.into_inner())
2608                .remove(module_id);
2609        }
2610    }
2611
2612    /// Whether `module_id` is a module this daemon supervises: on the roster,
2613    /// or about to be (its process is being spawned right now).
2614    ///
2615    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2616    /// for scopes: a supervised module's process can register and sync before
2617    /// [`Self::get`] can return it, and in that window it is still a module
2618    /// that will sync, not one that never will.
2619    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2620        self.configured_ids
2621            .lock()
2622            .unwrap_or_else(|poisoned| poisoned.into_inner())
2623            .contains(module_id)
2624    }
2625
2626    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2627        let modules = self
2628            .modules
2629            .lock()
2630            .unwrap_or_else(|poisoned| poisoned.into_inner());
2631        modules.get(module_id).cloned()
2632    }
2633
2634    pub(crate) fn record_late_health_answer(
2635        &self,
2636        module_id: &str,
2637        latency_ms: u64,
2638    ) -> Result<bool, SuperviseError> {
2639        let Some(module) = self.get(module_id) else {
2640            return Ok(false);
2641        };
2642        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2643            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2644            state.health.last_late_answer_latency_ms = Some(latency_ms);
2645            // A late answer is an answer: the module served the probe, just past
2646            // the deadline. Leaving the miss streak in place while logging
2647            // "proves the module is alive" is how a CPU-starved module that
2648            // answers every probe a few seconds late still marches to the
2649            // threshold and gets killed — the exact kill class `NoAnswer` is
2650            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2651            // is degradation, and degradation reports; it does not restart.
2652            state.health.consecutive_failures = 0;
2653        })?;
2654        Ok(true)
2655    }
2656
2657    /// Arm the one-shot marker for the module process that this caller
2658    /// deliberately initiated severance against. Generic connection teardown
2659    /// must not call this:
2660    /// a surviving process would otherwise retain an exemption for a later
2661    /// genuine crash.
2662    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2663        let Some(module) = self.get(module_id) else {
2664            return Ok(false);
2665        };
2666        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2667        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2668            return Ok(false);
2669        };
2670        drop(snapshot);
2671        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2672    }
2673
2674    pub fn list(&self) -> Vec<SupervisedModule> {
2675        let modules = self
2676            .modules
2677            .lock()
2678            .unwrap_or_else(|poisoned| poisoned.into_inner());
2679        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2680        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2681        modules
2682    }
2683
2684    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2685        self.activity_images
2686            .lock()
2687            .unwrap_or_else(|p| p.into_inner())
2688            .remove(module_id);
2689        self.spawn_nonces
2690            .lock()
2691            .unwrap_or_else(|poisoned| poisoned.into_inner())
2692            .remove(module_id);
2693        self.close_swap(module_id);
2694        let mut reserved_nonces = self
2695            .reserved_nonces
2696            .lock()
2697            .unwrap_or_else(|poisoned| poisoned.into_inner());
2698        if reserved_nonces.contains_key(module_id) {
2699            // The old nonce must die with the removed process, but the exact-id
2700            // gate remains until an operator explicitly releases it.
2701            reserved_nonces.insert(module_id.to_string(), None);
2702        }
2703        drop(reserved_nonces);
2704        self.reserved_prefix_owners
2705            .lock()
2706            .unwrap_or_else(|poisoned| poisoned.into_inner())
2707            .retain(|_, owner| owner != module_id);
2708        let removed = self
2709            .modules
2710            .lock()
2711            .unwrap_or_else(|poisoned| poisoned.into_inner())
2712            .remove(module_id);
2713        self.configured_ids
2714            .lock()
2715            .unwrap_or_else(|poisoned| poisoned.into_inner())
2716            .remove(module_id);
2717        removed
2718    }
2719
2720    /// Remember a module removed by a non-preview rescan so route.open can
2721    /// distinguish that intentional removal from an unknown id.
2722    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2723        self.removal_tombstones
2724            .lock()
2725            .unwrap_or_else(|poisoned| poisoned.into_inner())
2726            .insert(module_id.to_string(), unix_ms_now());
2727    }
2728
2729    /// Return how long ago a rescan removed this module in milliseconds.
2730    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2731        self.removal_tombstones
2732            .lock()
2733            .unwrap_or_else(|poisoned| poisoned.into_inner())
2734            .get(module_id)
2735            .copied()
2736            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2737    }
2738
2739    /// Retire a reserved-id gate only after its module has left supervision.
2740    ///
2741    /// A retained gate has no live nonce (`None`), so releasing any other entry
2742    /// would weaken a currently configured or otherwise active reservation.
2743    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2744        if self.get(module_id).is_some() {
2745            return false;
2746        }
2747        let mut reserved_nonces = self
2748            .reserved_nonces
2749            .lock()
2750            .unwrap_or_else(|poisoned| poisoned.into_inner());
2751        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2752            return false;
2753        }
2754        reserved_nonces.remove(module_id);
2755        true
2756    }
2757
2758    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2759        Arc::clone(&self.operation_lock)
2760    }
2761}
2762
2763/// Process supervisor for subc-owned singleton modules.
2764#[derive(Debug, Clone)]
2765pub struct Supervisor {
2766    registry: Arc<Registry>,
2767    restart_policy: RestartPolicy,
2768    drain_timeout: Duration,
2769    connection_file_path: Option<PathBuf>,
2770    capture_logs_dir: Option<PathBuf>,
2771    forwarding: Option<Arc<ForwardingTable>>,
2772    process_liveness: Arc<SupervisorProcessLiveness>,
2773    supervisor_handle: Option<SupervisorHandle>,
2774    health: HealthConfig,
2775    daemon_start_clock: crate::clock::StartClock,
2776    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2777    spawn_events: SpawnEventFeed,
2778    provenance_probe: ExecutableIdentityProbe,
2779    /// Every process spawned through this supervisor (and its clones) and not
2780    /// yet reaped, so daemon shutdown can end them.
2781    child_roster: ChildRoster,
2782    #[cfg(target_os = "linux")]
2783    cgroup_placement: Option<subc_cgroup::Placement>,
2784    #[cfg(test)]
2785    test_after_first_spawn: AfterFirstSpawnHook,
2786}
2787
2788/// A roster entry whose monitor and first process have not started yet.
2789/// Keeping the command receiver here lets discovery serve status immediately
2790/// while start/stop requests wait until bootstrap has published the connection file.
2791pub(crate) struct RegisteredConfiguredModule {
2792    spec: ModuleSpec,
2793    runtime: SupervisorRuntimeConfig,
2794    module: SupervisedModule,
2795    commands: mpsc::Receiver<SupervisorCommand>,
2796}
2797
2798/// Test-only hook run on the path that takes on a new module, right after its
2799/// first `spawn_child` returns (the process exists and could already be
2800/// registering) and before that process is handed to the module's supervise
2801/// loop and put on the roster. Lets a test observe what a fast child would see
2802/// in that window without racing a real one.
2803#[cfg(test)]
2804#[derive(Clone, Default)]
2805struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2806
2807#[cfg(test)]
2808type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2809
2810#[cfg(test)]
2811impl fmt::Debug for AfterFirstSpawnHook {
2812    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2813        f.write_str("AfterFirstSpawnHook")
2814    }
2815}
2816
2817#[cfg(test)]
2818impl AfterFirstSpawnHook {
2819    fn run(&self, module_id: &str) {
2820        if let Some(hook) = &self.0 {
2821            hook(module_id);
2822        }
2823    }
2824}
2825
2826impl Supervisor {
2827    #[cfg(test)]
2828    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2829        let supervisor = Self::new(registry, policy);
2830        #[cfg(target_os = "macos")]
2831        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2832        supervisor
2833    }
2834    /// Verify the trampoline once when configured. Missing private OS support
2835    /// refuses every macOS launch by name but does not stop the daemon's control
2836    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2837    /// entry point; the library must not exec an arbitrary hosting program.
2838    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2839        let path = path.into();
2840        #[cfg(target_os = "macos")]
2841        {
2842            let result = probe_privacy_trampoline(&path).map(|()| path);
2843            if let Err(cause) = &result {
2844                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2845            }
2846            self.child_roster.set_privacy_trampoline(result);
2847        }
2848        #[cfg(not(target_os = "macos"))]
2849        let _ = path;
2850        self
2851    }
2852    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2853    /// capability-requirement log to read.
2854    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2855        self.child_roster.shutdown_flag()
2856    }
2857
2858    /// The first step of an announced daemon shutdown, before the notice and
2859    /// before any connection is closed.
2860    ///
2861    /// Sets the daemon-shutdown flag first: from here on no module is
2862    /// respawned (crash restart, operator restart, or swap), and every child
2863    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2864    /// the module exits on the EOF this shutdown gives it or is signalled by a
2865    /// service manager that kills the whole cgroup. Then writes the journal's
2866    /// shutdown marker, which records the instant and closes this daemon
2867    /// incarnation's stretch of the journal.
2868    #[cfg(unix)]
2869    pub(crate) fn begin_daemon_shutdown(&self) {
2870        self.child_roster.close();
2871        if let Some(journal) = &self.terminal_journal {
2872            journal.stamp_shutdown();
2873        }
2874    }
2875
2876    /// Announce a cut while established connections can still carry replies.
2877    /// These budgets bound the attempt to deliver notices and the wait for
2878    /// routes to drain; neither guarantees delivery or child completion, and
2879    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2880    #[cfg(unix)]
2881    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2882        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2883        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2884        let Some(forwarding) = &self.forwarding else {
2885            return Ok(());
2886        };
2887        let notice_started = Instant::now();
2888        let modules = forwarding
2889            .begin_daemon_drain()
2890            .map_err(SuperviseError::Forwarding)?;
2891        let deadline_ms =
2892            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2893        let shutdown_reason = self
2894            .supervisor_handle
2895            .as_ref()
2896            .map(SupervisorHandle::take_daemon_shutdown_reason)
2897            .unwrap_or(RouteCloseReason::Disable);
2898        let mut modules_told = 0;
2899        let mut notices = tokio::task::JoinSet::new();
2900        let mut drains = Vec::new();
2901        for connection in forwarding
2902            .module_connections()
2903            .map_err(SuperviseError::Forwarding)?
2904        {
2905            let target = ModuleDrainTarget {
2906                endpoint: connection.endpoint,
2907                sink: connection.sink,
2908                negotiated_ver: connection.negotiated_ver,
2909                abandoned_bindings: Vec::new(),
2910                excluded_subscriptions: 0,
2911            };
2912            notices.spawn(async move {
2913                (
2914                    true,
2915                    send_module_draining(
2916                        &connection.module_id,
2917                        shutdown_reason,
2918                        deadline_ms,
2919                        &target,
2920                        true,
2921                    )
2922                    .await,
2923                )
2924            });
2925        }
2926        for (module_id, endpoint) in modules {
2927            let Some(target) = forwarding
2928                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2929                .map_err(SuperviseError::Forwarding)?
2930            else {
2931                continue;
2932            };
2933            let routes = forwarding
2934                .endpoint_routes(target.endpoint)
2935                .map_err(SuperviseError::Forwarding)?;
2936            // Only an announced restart promises that modules are coming back.
2937            // Client route notices retain their existing reconnect semantics.
2938            let mut recipients = Vec::new();
2939            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2940            for route in routes {
2941                let client = route.goodbye_target;
2942                if let Some((_, channels)) = clients
2943                    .iter_mut()
2944                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2945                {
2946                    channels.push(client.channel);
2947                } else {
2948                    let channel = client.channel;
2949                    clients.push((client, vec![channel]));
2950                }
2951            }
2952            for (client, mut channels) in clients {
2953                channels.sort_unstable();
2954                channels.dedup();
2955                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2956                    module_id: module_id.clone(),
2957                    channels,
2958                    reason: RouteCloseReason::Restart,
2959                })
2960                .expect("route closing serializes");
2961                recipients.push((client.sink, client.negotiated_ver, closing));
2962            }
2963            for (sink, version, body) in recipients {
2964                notices.spawn(async move {
2965                    let frame = Frame::build_with_version(
2966                        version,
2967                        FrameType::Push,
2968                        control_flags(),
2969                        0,
2970                        0,
2971                        0,
2972                        body,
2973                    )
2974                    .expect("bounded lifecycle notice frame builds");
2975                    (false, sink.send_flushed(frame).await.is_ok())
2976                });
2977            }
2978            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2979            drains.push((module_id, target.endpoint, gauges));
2980        }
2981        // A quiet forwarding table is not proof that queued notices reached the
2982        // socket. Wait for writer flush acknowledgements before testing quiescence.
2983        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2984        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2985            match result {
2986                Ok((true, true)) => modules_told += 1,
2987                Ok((false, true)) => {}
2988                result => warn!(?result, "daemon shutdown notice delivery failed"),
2989            }
2990        }
2991        notices.abort_all();
2992        info!(
2993            ?shutdown_reason,
2994            modules_told, "daemon shutdown module notices delivered"
2995        );
2996        let notice_elapsed = notice_started.elapsed();
2997        let drain_started = Instant::now();
2998        let deadline = drain_started + DRAIN_BUDGET;
2999        let mut waits = tokio::task::JoinSet::new();
3000        for (module_id, endpoint, gauges) in drains {
3001            let forwarding = Arc::clone(forwarding);
3002            let mut runtime = self.runtime_config();
3003            runtime.health.cadence = Duration::from_millis(100);
3004            waits.spawn(async move {
3005                let wait_result = wait_for_forwarding_quiescence(
3006                    &forwarding,
3007                    &module_id,
3008                    &runtime,
3009                    endpoint,
3010                    deadline,
3011                    &gauges,
3012                    DrainScope::Endpoint(endpoint),
3013                    &mut DrainGaugeTrace::new(&gauges),
3014                )
3015                .await;
3016                // Store the measurement before the task returns: quiescence
3017                // may be reached just as the deadline stops the join loop from
3018                // collecting results. Unfinished or failed waits report false.
3019                forwarding
3020                    .record_daemon_drain_result(
3021                        endpoint,
3022                        drained_after_quiescence_wait(&wait_result),
3023                    )
3024                    .map_err(SuperviseError::Forwarding)?;
3025                wait_result
3026            });
3027        }
3028        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3029            if !matches!(result, Ok(Ok(true))) {
3030                warn!(?result, "daemon shutdown drain did not reach quiescence");
3031            }
3032        }
3033        // Measure only the forwarding drain, not child teardown or the caller's
3034        // signal delivery and process-reaping delays.
3035        info!(
3036            notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3037            drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3038            "daemon shutdown forwarding drain complete"
3039        );
3040        Ok(())
3041    }
3042
3043    /// The last step of a graceful daemon shutdown, after the notice and the
3044    /// drain: send every registered module a module GOODBYE, the same planned
3045    /// stop signal `ck module stop` gives, then close every connection so each
3046    /// subc module sees EOF and starts its own teardown, then end every
3047    /// supervised child that has not exited
3048    /// by its own deadline (its drain budget, capped). Modules lead their own
3049    /// process groups, so a
3050    /// service manager's group kill no longer reaches them; without this a
3051    /// child that does not stop on EOF (every `protocol: "none"` child, which
3052    /// has no connection) would outlive the daemon. Every wait is bounded (see
3053    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
3054    #[cfg(unix)]
3055    pub(crate) async fn end_children_for_daemon_shutdown(
3056        &self,
3057        already_escalated: bool,
3058        escalate: impl std::future::Future<Output = ()>,
3059    ) {
3060        tokio::pin!(escalate);
3061        let mut escalated = already_escalated;
3062        if let Some(forwarding) = &self.forwarding {
3063            let reason = CloseReason::new(
3064                "daemon_shutdown",
3065                "the daemon is exiting after its shutdown notice and drain",
3066            );
3067            if escalated {
3068                // The operator asked to stop waiting: queue the GOODBYEs but
3069                // do not wait for them to be written.
3070                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3071            } else {
3072                tokio::select! {
3073                    biased;
3074                    _ = escalate.as_mut() => {
3075                        info!("second SIGTERM: abandoning module GOODBYE delivery");
3076                        escalated = true;
3077                    }
3078                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3079                }
3080            }
3081            let closed = forwarding.close_all_connections(&reason);
3082            debug!(closed, "closed established connections for daemon shutdown");
3083        }
3084        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
3085        // already completed and must not be polled again; the child shutdown
3086        // wait is told it is escalated and gets a future that never fires.
3087        let escalated_here = escalated && !already_escalated;
3088        let remaining_escalate = async move {
3089            if escalated_here {
3090                std::future::pending::<()>().await;
3091            } else {
3092                escalate.await;
3093            }
3094        };
3095        crate::child_roster::end_children_for_daemon_shutdown(
3096            &self.child_roster,
3097            escalated,
3098            remaining_escalate,
3099        )
3100        .await;
3101    }
3102
3103    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3104        Self {
3105            registry,
3106            restart_policy,
3107            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3108            connection_file_path: None,
3109            capture_logs_dir: None,
3110            forwarding: None,
3111            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3112            supervisor_handle: None,
3113            health: HealthConfig::default(),
3114            daemon_start_clock: crate::clock::StartClock::capture(),
3115            terminal_journal: None,
3116            spawn_events: SpawnEventFeed::default(),
3117            provenance_probe: ExecutableIdentityProbe::default(),
3118            child_roster: ChildRoster::default(),
3119            #[cfg(target_os = "linux")]
3120            cgroup_placement: None,
3121            #[cfg(test)]
3122            test_after_first_spawn: AfterFirstSpawnHook::default(),
3123        }
3124    }
3125
3126    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3127        self.drain_timeout = drain_timeout;
3128        self
3129    }
3130
3131    /// The drain budget a module gets when its config entry sets none.
3132    pub(crate) fn default_drain_timeout(&self) -> Duration {
3133        self.drain_timeout
3134    }
3135
3136    pub fn with_process_liveness(
3137        mut self,
3138        process_liveness: Arc<SupervisorProcessLiveness>,
3139    ) -> Self {
3140        self.process_liveness = process_liveness;
3141        self
3142    }
3143
3144    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3145        self.connection_file_path = Some(connection_file_path.into());
3146        self
3147    }
3148
3149    /// Enables daemon-owned capture files for supervised stdout and stderr.
3150    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3151        self.capture_logs_dir = Some(logs_dir.into());
3152        self
3153    }
3154
3155    /// Names this daemon lifetime in spawn events, independently of whether a
3156    /// terminal journal is configured.
3157    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3158        // A millisecond start stamp can repeat after clock rollback or a rapid
3159        // restart. Use the connection file's random daemon_id instead: it already
3160        // identifies this daemon lifetime independently of the wall clock.
3161        self.spawn_events.configure_incarnation(daemon_incarnation);
3162        self
3163    }
3164
3165    /// Enables best-effort history shared by every supervised module. Without
3166    /// it, terminal history is kept only in each module's in-memory ring.
3167    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3168        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3169        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3170            path,
3171            daemon_incarnation,
3172        )));
3173        this
3174    }
3175
3176    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3177        self.forwarding = Some(forwarding);
3178        self
3179    }
3180
3181    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3182        self.spawn_events = supervisor_handle.spawn_events.clone();
3183        self.supervisor_handle = Some(supervisor_handle);
3184        self
3185    }
3186
3187    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3188        self.health = health;
3189        self
3190    }
3191
3192    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3193    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3194    /// record is kept.
3195    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3196        self.child_roster.record_to(path.into());
3197        self
3198    }
3199
3200    #[cfg(target_os = "linux")]
3201    pub fn with_cgroup_placement(
3202        mut self,
3203        cgroup_placement: Option<subc_cgroup::Placement>,
3204    ) -> Self {
3205        self.cgroup_placement = cgroup_placement;
3206        self
3207    }
3208
3209    /// Spawn `spec.program` and start monitoring it.
3210    ///
3211    /// The child is expected to parse `--subc <connection-file-path>`, read the
3212    /// TCP+key connection file, authenticate to the already-running listener, and
3213    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3214    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3215        validate_spec(&spec)?;
3216        self.establish_identity(&spec);
3217
3218        let runtime = self.runtime_config();
3219        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3220        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3221        let registration_watch = watch_registration(&self.registry, &spec);
3222        let spawned = spawn_child(
3223            &spec,
3224            runtime.connection_file_path.as_deref(),
3225            self.supervisor_handle.as_ref(),
3226            &runtime.stderr_ring,
3227            runtime.capture_logs_dir.as_deref(),
3228            &runtime.child_roster,
3229            #[cfg(target_os = "linux")]
3230            runtime.cgroup_placement.as_ref(),
3231        );
3232        #[cfg(test)]
3233        self.test_after_first_spawn.run(&spec.module_id);
3234        let mut child = match spawned {
3235            Ok(child) => child,
3236            Err(err) => {
3237                // Unlike the configured paths, a failed `spawn` leaves nothing
3238                // on the roster, so the module must not stay marked configured.
3239                self.abandon_unrostered(&spec.module_id);
3240                return Err(err);
3241            }
3242        };
3243        arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3244        self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3245        if let Some(spawn) = activity_spawn {
3246            spawn.complete(&snapshot);
3247        }
3248
3249        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3250    }
3251
3252    /// Make `spec`'s module count as configured, with its identity gates
3253    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3254    /// exists.
3255    ///
3256    /// Every path that takes on a new module calls this before `spawn_child`.
3257    /// The order is the point: the child can connect, register, sync its
3258    /// scopes and ask about them as soon as it is spawned, and the module is
3259    /// only put on the roster after `spawn_child` returns. Were the mark set
3260    /// with the roster entry, a fast child would see its own owner reported
3261    /// as not configured, and a scoped `route.open` in that window would be
3262    /// refused as terminal `scope_not_live` ("will never sync") instead of
3263    /// retryable `scope_not_synced`.
3264    fn establish_identity(&self, spec: &ModuleSpec) {
3265        if let Some(supervisor_handle) = &self.supervisor_handle {
3266            supervisor_handle.apply_identity_configuration(spec);
3267            supervisor_handle.mark_configured(&spec.module_id);
3268        }
3269    }
3270
3271    /// Take back [`Self::establish_identity`]'s configured mark when the
3272    /// module will not be put on the roster after all.
3273    fn abandon_unrostered(&self, module_id: &str) {
3274        if let Some(supervisor_handle) = &self.supervisor_handle {
3275            supervisor_handle.unmark_configured_unless_rostered(module_id);
3276        }
3277    }
3278
3279    /// Record a freshly spawned first process as running. On failure the
3280    /// module never reaches the roster, so its configured mark is taken back.
3281    fn mark_first_process_running(
3282        &self,
3283        spec: &ModuleSpec,
3284        runtime: &SupervisorRuntimeConfig,
3285        snapshot: &SharedSnapshot,
3286        child: &mut SupervisedChild,
3287    ) -> Result<(), SuperviseError> {
3288        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3289            self.abandon_unrostered(&spec.module_id);
3290            return Err(err);
3291        }
3292        self.process_liveness
3293            .track(spec.module_id.clone(), Arc::clone(snapshot));
3294        Ok(())
3295    }
3296
3297    /// Start supervising a module declared in daemon configuration.
3298    ///
3299    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3300    /// failures in the supervisor handle so operator-facing `supervisor.list`
3301    /// reflects every configured module while daemon startup continues.
3302    pub fn supervise_configured(
3303        &self,
3304        spec: ModuleSpec,
3305        enabled: bool,
3306    ) -> Result<SupervisedModule, SuperviseError> {
3307        self.supervise_configured_requested(
3308            spec,
3309            enabled,
3310            self.health.clone(),
3311            None,
3312            self.restart_policy,
3313            DEFAULT_REGISTER_TIMEOUT,
3314            None,
3315        )
3316    }
3317
3318    /// Supervise a configured module with its own health, drain, and crash
3319    /// budget. The restart policy is per-module because the config file is:
3320    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3321    /// module that is expensive to restart should not be forced onto the same
3322    /// budget as one that is cheap.
3323    pub fn supervise_configured_with_health(
3324        &self,
3325        spec: ModuleSpec,
3326        enabled: bool,
3327        health: HealthConfig,
3328        drain_timeout_ms: Option<u64>,
3329        restart_policy: RestartPolicy,
3330    ) -> Result<SupervisedModule, SuperviseError> {
3331        self.supervise_configured_requested(
3332            spec,
3333            enabled,
3334            health,
3335            drain_timeout_ms,
3336            restart_policy,
3337            DEFAULT_REGISTER_TIMEOUT,
3338            None,
3339        )
3340    }
3341
3342    #[allow(clippy::too_many_arguments)]
3343    pub(crate) fn supervise_configured_requested(
3344        &self,
3345        spec: ModuleSpec,
3346        enabled: bool,
3347        health: HealthConfig,
3348        drain_timeout_ms: Option<u64>,
3349        restart_policy: RestartPolicy,
3350        register_timeout: Duration,
3351        requester: Option<Principal>,
3352    ) -> Result<SupervisedModule, SuperviseError> {
3353        let registered = self.register_configured_with_health(
3354            spec,
3355            enabled,
3356            health,
3357            drain_timeout_ms,
3358            restart_policy,
3359            register_timeout,
3360        )?;
3361        let daemon_initiated = requester.is_none();
3362        self.start_registered_configured(registered, requester, daemon_initiated)
3363    }
3364
3365    /// Record a configured module as starting or disabled, without launching a
3366    /// process or running its monitor. Bootstrap registers the entire roster
3367    /// before publishing discovery, then activates it once children can read
3368    /// the connection file passed in their `--subc` argument.
3369    pub(crate) fn register_configured_with_health(
3370        &self,
3371        spec: ModuleSpec,
3372        enabled: bool,
3373        health: HealthConfig,
3374        drain_timeout_ms: Option<u64>,
3375        restart_policy: RestartPolicy,
3376        register_timeout: Duration,
3377    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3378        validate_spec(&spec)?;
3379        self.establish_identity(&spec);
3380
3381        let mut runtime = self.runtime_config();
3382        runtime.health = health;
3383        runtime.restart_policy = restart_policy;
3384        runtime.register_timeout = register_timeout;
3385        if let Some(ms) = drain_timeout_ms {
3386            runtime.drain_timeout = Duration::from_millis(ms);
3387            *runtime
3388                .effective_drain_timeout
3389                .lock()
3390                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3391        }
3392        let snapshot = if enabled {
3393            SupervisorSnapshot::starting()
3394        } else {
3395            SupervisorSnapshot::disabled()
3396        };
3397        let snapshot = {
3398            let mut snapshot = snapshot;
3399            if enabled {
3400                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3401                {
3402                    snapshot.waiting_for_previous_pid = Some(pid);
3403                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3404                }
3405            }
3406            snapshot
3407        };
3408        let (module, commands) =
3409            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3410        Ok(RegisteredConfiguredModule {
3411            spec,
3412            runtime,
3413            module,
3414            commands,
3415        })
3416    }
3417
3418    /// Activate a previously registered module. Queued control requests are
3419    /// served by the monitor after the initial configured spawn has settled.
3420    pub(crate) fn start_registered_configured(
3421        &self,
3422        registered: RegisteredConfiguredModule,
3423        requester: Option<Principal>,
3424        daemon_initiated: bool,
3425    ) -> Result<SupervisedModule, SuperviseError> {
3426        let RegisteredConfiguredModule {
3427            spec,
3428            mut runtime,
3429            module,
3430            commands,
3431        } = registered;
3432        runtime.start_requester = requester;
3433        runtime.recovery_is_daemon = daemon_initiated;
3434        let snapshot = &module.inner.snapshot;
3435        let should_spawn = {
3436            let state = lock_snapshot(snapshot)?;
3437            state.enabled && state.waiting_for_previous_pid.is_none()
3438        };
3439        let child = if should_spawn {
3440            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3441            let registration_watch = watch_registration(&self.registry, &spec);
3442            let spawned = spawn_child(
3443                &spec,
3444                runtime.connection_file_path.as_deref(),
3445                self.supervisor_handle.as_ref(),
3446                &runtime.stderr_ring,
3447                runtime.capture_logs_dir.as_deref(),
3448                &runtime.child_roster,
3449                #[cfg(target_os = "linux")]
3450                runtime.cgroup_placement.as_ref(),
3451            );
3452            #[cfg(test)]
3453            self.test_after_first_spawn.run(&spec.module_id);
3454            match spawned {
3455                Ok(mut child) => {
3456                    arm_registration_deadline(
3457                        &mut child,
3458                        runtime.register_timeout,
3459                        registration_watch,
3460                    );
3461                    self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3462                    if let Some(spawn) = activity_spawn {
3463                        spawn.complete(snapshot);
3464                    }
3465                    Some(child)
3466                }
3467                Err(err) => {
3468                    if runtime.health.critical {
3469                        error!(
3470                            module_id = %spec.module_id,
3471                            program = %spec.program.display(),
3472                            error = %err,
3473                            "critical configured module failed to spawn; marking failed and alerting"
3474                        );
3475                    } else {
3476                        error!(
3477                            module_id = %spec.module_id,
3478                            program = %spec.program.display(),
3479                            error = %err,
3480                            "configured module failed to spawn; marking failed and continuing"
3481                        );
3482                    }
3483                    fail_snapshot(
3484                        snapshot,
3485                        Some(&spec.module_id),
3486                        None,
3487                        &runtime.failure_context(daemon_initiated),
3488                    );
3489                    None
3490                }
3491            }
3492        } else {
3493            None
3494        };
3495        self.start_monitor(spec, runtime, &module, child, commands);
3496        Ok(module)
3497    }
3498
3499    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3500        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3501        SupervisorRuntimeConfig {
3502            registry: self.registry.clone(),
3503            recovery_is_daemon: true,
3504            start_requester: None,
3505            pending_activity_restart: false,
3506            provenance_probe: self.provenance_probe.clone(),
3507            scheduled_respawn: Arc::default(),
3508            deferred_reload_reply: Arc::default(),
3509            restart_policy: self.restart_policy,
3510            register_timeout: DEFAULT_REGISTER_TIMEOUT,
3511            drain_timeout: self.drain_timeout,
3512            // Shared with this module's roster copy: daemon shutdown waits on
3513            // each child for the module's own drain budget, as resolved now.
3514            child_roster: self
3515                .child_roster
3516                .for_module(Arc::clone(&effective_drain_timeout)),
3517            effective_drain_timeout,
3518            default_drain_timeout: self.drain_timeout,
3519            health: self.health.clone(),
3520            connection_file_path: self.connection_file_path.clone(),
3521            capture_logs_dir: self.capture_logs_dir.clone(),
3522            forwarding: self.forwarding.clone(),
3523            supervisor_handle: self.supervisor_handle.clone(),
3524            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3525            terminal_ring: Arc::new(Mutex::new(
3526                TerminalRing::new(
3527                    TerminalRingConfig::default(),
3528                    self.daemon_start_clock.started_at_ms(),
3529                )
3530                .with_start_clock(self.daemon_start_clock)
3531                .with_journal(self.terminal_journal.clone())
3532                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3533            )),
3534            spawn_events: self.spawn_events.clone(),
3535            #[cfg(target_os = "linux")]
3536            cgroup_placement: self.cgroup_placement.clone(),
3537            #[cfg(test)]
3538            test_seed_stale_facts_before_enable_spawn: false,
3539            #[cfg(test)]
3540            test_reload_exit_record_gate: None,
3541        }
3542    }
3543
3544    fn supervised_module(
3545        &self,
3546        spec: ModuleSpec,
3547        runtime: SupervisorRuntimeConfig,
3548        snapshot: SharedSnapshot,
3549        child: Option<SupervisedChild>,
3550    ) -> SupervisedModule {
3551        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3552        self.start_monitor(spec, runtime, &module, child, commands);
3553        module
3554    }
3555
3556    fn register_module(
3557        &self,
3558        spec: &ModuleSpec,
3559        runtime: &SupervisorRuntimeConfig,
3560        snapshot: SharedSnapshot,
3561    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3562        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3563            spec: spec.clone(),
3564            health: runtime.health.clone(),
3565        }));
3566        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3567        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3568        // The module's OWN policy, which may be its per-module config rather than
3569        // the supervisor-wide one; status must report the budget the supervise
3570        // loop actually enforces.
3571        let restart_policy = runtime.restart_policy;
3572        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3573        let (tx, rx) = mpsc::channel(4);
3574        let module_id = spec.module_id.clone();
3575        let module = SupervisedModule {
3576            inner: Arc::new(SupervisedModuleInner {
3577                module_id: module_id.clone(),
3578                registry: Arc::clone(&self.registry),
3579                snapshot,
3580                configuration,
3581                stderr_ring,
3582                terminal_ring,
3583                commands: tx,
3584                monitor: Mutex::new(None),
3585                restart_policy,
3586                effective_drain_timeout,
3587                provenance_probe: self.provenance_probe.clone(),
3588            }),
3589        };
3590        // Identity gates and the configured mark must precede the roster entry,
3591        // so a client never sees a module without its admission policy.
3592        if let Some(supervisor_handle) = &self.supervisor_handle {
3593            supervisor_handle.insert(module.clone());
3594        }
3595        (module, rx)
3596    }
3597
3598    fn start_monitor(
3599        &self,
3600        spec: ModuleSpec,
3601        runtime: SupervisorRuntimeConfig,
3602        module: &SupervisedModule,
3603        child: Option<SupervisedChild>,
3604        commands: mpsc::Receiver<SupervisorCommand>,
3605    ) {
3606        let monitor = tokio::spawn(supervise_loop(
3607            spec,
3608            runtime,
3609            Arc::clone(&self.registry),
3610            Arc::clone(&self.process_liveness),
3611            Arc::clone(&module.inner.snapshot),
3612            child,
3613            commands,
3614        ));
3615        *module
3616            .inner
3617            .monitor
3618            .lock()
3619            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3620    }
3621}
3622
3623impl Default for Supervisor {
3624    fn default() -> Self {
3625        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3626    }
3627}
3628
3629/// Handle to one supervised child process.
3630#[derive(Clone)]
3631pub struct SupervisedModule {
3632    inner: Arc<SupervisedModuleInner>,
3633}
3634
3635struct SupervisedModuleInner {
3636    module_id: String,
3637    registry: Arc<Registry>,
3638    snapshot: SharedSnapshot,
3639    configuration: Arc<Mutex<SupervisedConfiguration>>,
3640    stderr_ring: Arc<Mutex<StderrRing>>,
3641    terminal_ring: Arc<Mutex<TerminalRing>>,
3642    commands: mpsc::Sender<SupervisorCommand>,
3643    monitor: Mutex<Option<JoinHandle<()>>>,
3644    /// Copied from the supervisor's runtime config at spawn so `status()` can
3645    /// report the restart budget without reaching back into the supervisor. The
3646    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3647    restart_policy: RestartPolicy,
3648    effective_drain_timeout: Arc<Mutex<Duration>>,
3649    provenance_probe: ExecutableIdentityProbe,
3650}
3651
3652impl fmt::Debug for SupervisedModule {
3653    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3654        f.debug_struct("SupervisedModule")
3655            .field("module_id", &self.inner.module_id)
3656            .field("status", &self.status())
3657            .finish_non_exhaustive()
3658    }
3659}
3660
3661impl SupervisedModule {
3662    pub fn module_id(&self) -> &str {
3663        &self.inner.module_id
3664    }
3665
3666    /// Test-only: put one probe miss on the streak, the way
3667    /// `handle_health_probe_failure` does, so tests can assert what a later
3668    /// event does to the streak without driving the whole probe loop.
3669    #[cfg(test)]
3670    pub(crate) fn record_health_probe_failure_for_test(
3671        &self,
3672        detail: &str,
3673    ) -> Result<(), SuperviseError> {
3674        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3675            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3676            state.health.detail = Some(detail.to_string());
3677        })
3678    }
3679
3680    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3681        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3682    }
3683
3684    pub(crate) fn registration_end_reason(
3685        &self,
3686    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3687        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3688        Ok(match snapshot.state {
3689            ModuleState::Draining if snapshot.draining_to_replace => {
3690                Some(RegistrationEndReason::SupervisorRestart)
3691            }
3692            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3693            ModuleState::Restarting if snapshot.draining_to_replace => {
3694                Some(RegistrationEndReason::SupervisorRestart)
3695            }
3696            _ => None,
3697        })
3698    }
3699
3700    /// The module's retained stderr, newest lines last.
3701    ///
3702    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3703    /// module, `supervisor.list` renders every module, and putting it in the
3704    /// shared snapshot would make each status read carry a payload almost nobody
3705    /// asked for. Callers that want the text ask for it.
3706    pub fn stderr_tail(
3707        &self,
3708        max_lines: Option<usize>,
3709        max_bytes: Option<usize>,
3710    ) -> StderrTailSnapshot {
3711        self.inner
3712            .stderr_ring
3713            .lock()
3714            .unwrap_or_else(|poisoned| poisoned.into_inner())
3715            .snapshot(max_lines, max_bytes)
3716    }
3717
3718    /// The module's bounded terminal history, oldest retained exit first.
3719    ///
3720    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3721    /// daemon whose in-memory history was necessarily reset.
3722    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3723        self.inner
3724            .terminal_ring
3725            .lock()
3726            .unwrap_or_else(|poisoned| poisoned.into_inner())
3727            .snapshot()
3728    }
3729
3730    /// Retained observations from the current ring and all journal generations.
3731    ///
3732    /// Blocking: this reads the journal files. Async callers use
3733    /// `Self::read_durable_terminal_history`.
3734    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3735        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3736    }
3737
3738    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3739    /// read (up to every retained generation) never occupies a runtime worker.
3740    /// Fails only if the blocking task could not finish (runtime shutdown or a
3741    /// panic in the read).
3742    pub(crate) async fn read_durable_terminal_history(
3743        &self,
3744    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3745        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3746        let module_id = self.inner.module_id.clone();
3747        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3748            .await
3749    }
3750
3751    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3752        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3753            .map(|(status, _)| status)
3754    }
3755
3756    pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3757        let state = lock_snapshot(&self.inner.snapshot)?;
3758        Ok(state
3759            .failed_retry
3760            .pending
3761            .filter(|_| state.state == ModuleState::Failed && state.enabled)
3762            .map(|retry| {
3763                unix_ms_now().saturating_add(
3764                    u64::try_from(
3765                        retry
3766                            .deadline
3767                            .saturating_duration_since(Instant::now())
3768                            .as_millis(),
3769                    )
3770                    .unwrap_or(u64::MAX),
3771                )
3772            }))
3773    }
3774
3775    pub(crate) fn record_deliberate_severance(
3776        &self,
3777        identity: ProcessIdentity,
3778    ) -> Result<bool, SuperviseError> {
3779        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3780        if snapshot.pid != Some(identity.pid)
3781            || snapshot.process_start_time != Some(identity.start_time)
3782        {
3783            return Ok(false);
3784        }
3785        snapshot.deliberate_severance = Some(identity);
3786        Ok(true)
3787    }
3788
3789    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3790    ///
3791    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3792    /// does not produce reader-observability logs.
3793    pub(crate) fn status_for_control(
3794        &self,
3795        caller: &'static str,
3796    ) -> Result<ModuleStatus, SuperviseError> {
3797        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3798            .map(|(status, _)| status)
3799    }
3800
3801    fn status_with_snapshot_lock(
3802        &self,
3803        snapshot: &SharedSnapshot,
3804        caller: Option<&'static str>,
3805    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3806        let mut guard = match caller {
3807            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3808            None => lock_snapshot(snapshot)?,
3809        };
3810        // Read the budget through the pruning path so a reader sees the same
3811        // in-window count the restart decision would use, not a stale total.
3812        let restart_count =
3813            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3814        let snapshot = guard.clone();
3815        drop(guard);
3816        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3817            SuperviseError::StatePoisoned {
3818                module_id: Some(self.inner.module_id.clone()),
3819            }
3820        })?;
3821        let registration_active = self
3822            .inner
3823            .registry
3824            .get_module(&self.inner.module_id)
3825            .map_err(SuperviseError::Registry)?
3826            .is_some();
3827        let protocol = snapshot
3828            .spawned_protocol
3829            .unwrap_or(self.declared_protocol()?);
3830        let running_process =
3831            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3832        // Registration is the difference between the two protocols and the only
3833        // one: a subc module that has not registered cannot serve a request even
3834        // though its process is up, and a `none` module never registers at all,
3835        // so requiring it there would pin `live` to false for the whole life of
3836        // a perfectly healthy process.
3837        let live = match protocol {
3838            ModuleProtocol::Subc => running_process && registration_active,
3839            ModuleProtocol::None => running_process,
3840        };
3841
3842        Ok((
3843            ModuleStatus {
3844                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3845                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3846                module_id: self.inner.module_id.clone(),
3847                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3848                state: snapshot.state,
3849                enabled: snapshot.enabled,
3850                process_alive: snapshot.process_alive,
3851                registration_active,
3852                protocol,
3853                live,
3854                restart_count,
3855                lifetime_restarts: snapshot.lifetime_restarts,
3856                spawn_generation: snapshot.spawn_generation,
3857                max_restarts: self.inner.restart_policy.max_restarts,
3858                restart_window: self.inner.restart_policy.window,
3859                drain_timeout,
3860                restart_backoff: self.inner.restart_policy.backoff,
3861                restart_max_backoff: self.inner.restart_policy.max_backoff,
3862                pid: snapshot.reported_pid(),
3863                spawned_at_ms: snapshot.spawned_at_ms,
3864                spawned_from: snapshot.spawned_from,
3865                launch_spec_digest: snapshot.launch_spec_digest,
3866                process_start_time: snapshot.process_start_time,
3867                last_exit: snapshot.last_exit,
3868                health: snapshot.health,
3869            },
3870            snapshot.spawned_file_identity.clone(),
3871        ))
3872    }
3873
3874    #[cfg(test)]
3875    pub(crate) fn hold_snapshot_for_test(
3876        &self,
3877        acquired: std::sync::mpsc::Sender<()>,
3878        hold: Duration,
3879    ) -> std::thread::JoinHandle<()> {
3880        let snapshot = Arc::clone(&self.inner.snapshot);
3881        std::thread::spawn(move || {
3882            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3883            acquired
3884                .send(())
3885                .expect("test receiver waits for snapshot lock");
3886            std::thread::sleep(hold);
3887        })
3888    }
3889
3890    /// The status and the running-image check for `supervisor.provenance`,
3891    /// taken from one status read. The exec acknowledgement can land between
3892    /// two separate reads, and the reply would then pair "no pid yet" with an
3893    /// image observed after the module started, which describes no single
3894    /// moment.
3895    pub(crate) async fn status_and_running_image_agreement(
3896        &self,
3897    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3898        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3899        let image = self
3900            .inner
3901            .provenance_probe
3902            .observe(
3903                status.pid,
3904                status.spawned_from.as_deref(),
3905                identity,
3906                status.process_start_time,
3907            )
3908            .await;
3909        Ok((status, image))
3910    }
3911
3912    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3913        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3914            Ok(snapshot) => snapshot.clone(),
3915            Err(_) => {
3916                return subc_control::RunningImageAgreement::Unavailable {
3917                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3918                };
3919            }
3920        };
3921        self.inner
3922            .provenance_probe
3923            .observe(
3924                snapshot.reported_pid(),
3925                snapshot.spawned_from.as_deref(),
3926                snapshot.spawned_file_identity.clone(),
3927                snapshot.process_start_time,
3928            )
3929            .await
3930    }
3931
3932    /// Memory and CPU time of the module's current process, read now. Only the
3933    /// process the supervisor spawned is read, not processes it has started.
3934    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3935        #[cfg(windows)]
3936        {
3937            let image = lock_snapshot(&self.inner.snapshot)
3938                .ok()
3939                .and_then(|snapshot| {
3940                    snapshot
3941                        .spawned_file_identity
3942                        .as_ref()
3943                        .and_then(|identity| identity.windows_image.clone())
3944                });
3945            match image {
3946                Some(image) => {
3947                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3948                }
3949                None => subc_control::ChildResourceUsage::Unavailable {
3950                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3951                },
3952            }
3953        }
3954        #[cfg(not(windows))]
3955        {
3956            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3957                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3958                Err(_) => {
3959                    return subc_control::ChildResourceUsage::Unavailable {
3960                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3961                    }
3962                }
3963            };
3964            crate::child_resources::read(pid, start_time)
3965        }
3966    }
3967
3968    pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
3969        self.inner
3970            .snapshot
3971            .try_lock()
3972            .map(|snapshot| snapshot.subtree_target.clone())
3973            .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
3974    }
3975
3976    #[cfg(test)]
3977    pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
3978        self.inner.snapshot.lock().unwrap().subtree_target = target;
3979    }
3980
3981    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3982        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3983        Ok(match snapshot.state {
3984            ModuleState::Restarting => true,
3985            ModuleState::Failed | ModuleState::Disabled => false,
3986            _ if snapshot.enabled && snapshot.respawn_pending => true,
3987            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3988        })
3989    }
3990
3991    #[cfg(test)]
3992    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3993        self.is_warming_with_snapshot_lock(None)
3994    }
3995
3996    pub(crate) fn is_warming_for_control(
3997        &self,
3998        caller: &'static str,
3999    ) -> Result<bool, SuperviseError> {
4000        self.is_warming_with_snapshot_lock(Some(caller))
4001    }
4002
4003    fn is_warming_with_snapshot_lock(
4004        &self,
4005        caller: Option<&'static str>,
4006    ) -> Result<bool, SuperviseError> {
4007        let snapshot = match caller {
4008            Some(caller) => {
4009                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4010            }
4011            None => lock_snapshot(&self.inner.snapshot)?,
4012        }
4013        .clone();
4014        Ok(matches!(
4015            snapshot.state,
4016            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4017        ))
4018    }
4019
4020    /// Drain the module and stop monitoring it.
4021    pub async fn drain(&self) -> Result<(), SuperviseError> {
4022        self.stop().await
4023    }
4024
4025    pub(crate) async fn retire_requested(
4026        &self,
4027        requester: Option<Principal>,
4028    ) -> Result<(), SuperviseError> {
4029        match self.state()? {
4030            ModuleState::Stopped => return Ok(()),
4031            ModuleState::Failed
4032                if lock_snapshot(&self.inner.snapshot)?
4033                    .failed_retry
4034                    .pending
4035                    .is_none() =>
4036            {
4037                return Ok(())
4038            }
4039            ModuleState::Failed
4040            | ModuleState::Starting
4041            | ModuleState::Running
4042            | ModuleState::Unresponsive
4043            | ModuleState::Restarting
4044            | ModuleState::Draining
4045            | ModuleState::Disabled => {}
4046        }
4047
4048        let (reply_tx, reply_rx) = oneshot::channel();
4049        self.inner
4050            .commands
4051            .send(SupervisorCommand::Retire {
4052                requester,
4053                reply: reply_tx,
4054            })
4055            .await
4056            .map_err(|_| SuperviseError::CommandClosed {
4057                module_id: self.inner.module_id.clone(),
4058            })?;
4059        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4060            module_id: self.inner.module_id.clone(),
4061        })?
4062    }
4063
4064    pub async fn stop(&self) -> Result<(), SuperviseError> {
4065        self.stop_requested(None).await
4066    }
4067
4068    pub(crate) async fn stop_requested(
4069        &self,
4070        requester: Option<Principal>,
4071    ) -> Result<(), SuperviseError> {
4072        match self.state()? {
4073            ModuleState::Stopped => return Ok(()),
4074            ModuleState::Failed
4075                if lock_snapshot(&self.inner.snapshot)?
4076                    .failed_retry
4077                    .pending
4078                    .is_none() =>
4079            {
4080                return Ok(())
4081            }
4082            ModuleState::Failed
4083            | ModuleState::Starting
4084            | ModuleState::Running
4085            | ModuleState::Unresponsive
4086            | ModuleState::Restarting
4087            | ModuleState::Draining
4088            | ModuleState::Disabled => {}
4089        }
4090
4091        let (reply_tx, reply_rx) = oneshot::channel();
4092        self.inner
4093            .commands
4094            .send(SupervisorCommand::Drain {
4095                requester,
4096                reply: reply_tx,
4097            })
4098            .await
4099            .map_err(|_| SuperviseError::CommandClosed {
4100                module_id: self.inner.module_id.clone(),
4101            })?;
4102        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4103            module_id: self.inner.module_id.clone(),
4104        })?
4105    }
4106
4107    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4108        self.restart_requested(drain_timeout_ms, None).await
4109    }
4110
4111    pub(crate) async fn restart_requested(
4112        &self,
4113        drain_timeout_ms: Option<u64>,
4114        requester: Option<Principal>,
4115    ) -> Result<(), SuperviseError> {
4116        let queued_at = Instant::now();
4117        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4118        let (reply_tx, reply_rx) = oneshot::channel();
4119        self.inner
4120            .commands
4121            .send(SupervisorCommand::Restart {
4122                requester,
4123                drain_timeout_ms,
4124                received_at_generation,
4125                queued_at,
4126                reply: reply_tx,
4127            })
4128            .await
4129            .map_err(|_| SuperviseError::CommandClosed {
4130                module_id: self.inner.module_id.clone(),
4131            })?;
4132        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4133            module_id: self.inner.module_id.clone(),
4134        })?
4135    }
4136
4137    /// Blue/green restart: see `SupervisorCommand::Swap` and the
4138    /// `supervisor_swap` module. Returns once the swap has cut over (the old
4139    /// process then drains in the background of the supervise loop) or has
4140    /// failed, leaving the old process serving.
4141    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4142        self.swap_requested(ready_timeout, None).await
4143    }
4144
4145    pub(crate) async fn swap_requested(
4146        &self,
4147        ready_timeout: Option<Duration>,
4148        requester: Option<Principal>,
4149    ) -> Result<(), SuperviseError> {
4150        let (reply_tx, reply_rx) = oneshot::channel();
4151        self.inner
4152            .commands
4153            .send(SupervisorCommand::Swap {
4154                requester,
4155                ready_timeout,
4156                reply: reply_tx,
4157            })
4158            .await
4159            .map_err(|_| SuperviseError::CommandClosed {
4160                module_id: self.inner.module_id.clone(),
4161            })?;
4162        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4163            module_id: self.inner.module_id.clone(),
4164        })?
4165    }
4166
4167    pub async fn reload(&self) -> Result<(), SuperviseError> {
4168        self.reload_requested(None).await
4169    }
4170
4171    pub(crate) async fn reload_requested(
4172        &self,
4173        requester: Option<Principal>,
4174    ) -> Result<(), SuperviseError> {
4175        let (reply_tx, reply_rx) = oneshot::channel();
4176        self.inner
4177            .commands
4178            .send(SupervisorCommand::Reload {
4179                requester,
4180                reply: reply_tx,
4181            })
4182            .await
4183            .map_err(|_| SuperviseError::CommandClosed {
4184                module_id: self.inner.module_id.clone(),
4185            })?;
4186        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4187            module_id: self.inner.module_id.clone(),
4188        })?
4189    }
4190
4191    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4192        self.set_enabled_requested(enabled, None).await
4193    }
4194
4195    pub(crate) async fn set_enabled_requested(
4196        &self,
4197        enabled: bool,
4198        requester: Option<Principal>,
4199    ) -> Result<bool, SuperviseError> {
4200        let (reply_tx, reply_rx) = oneshot::channel();
4201        self.inner
4202            .commands
4203            .send(SupervisorCommand::SetEnabled {
4204                requester,
4205                enabled,
4206                reply: reply_tx,
4207            })
4208            .await
4209            .map_err(|_| SuperviseError::CommandClosed {
4210                module_id: self.inner.module_id.clone(),
4211            })?;
4212        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4213            module_id: self.inner.module_id.clone(),
4214        })?
4215    }
4216
4217    /// The current process's protocol, or the configured protocol when down.
4218    /// A rescan stores the next launch spec without changing how an existing
4219    /// process registers, serves routes, is probed, or exits.
4220    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4221        let configured = self
4222            .inner
4223            .configuration
4224            .lock()
4225            .map_err(|_| SuperviseError::StatePoisoned {
4226                module_id: Some(self.inner.module_id.clone()),
4227            })?
4228            .spec
4229            .protocol;
4230        let state = lock_snapshot(&self.inner.snapshot)?;
4231        Ok(state.spawned_protocol.unwrap_or(configured))
4232    }
4233
4234    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4235        let configuration =
4236            self.inner
4237                .configuration
4238                .lock()
4239                .map_err(|_| SuperviseError::StatePoisoned {
4240                    module_id: Some(self.inner.module_id.clone()),
4241                })?;
4242        Ok((configuration.spec.clone(), configuration.health.clone()))
4243    }
4244
4245    /// Replace this module's launch spec, keeping its health and drain policy,
4246    /// the way a rescan does for a changed config entry. The running process is
4247    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4248    #[cfg(any(test, feature = "test-support"))]
4249    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4250        let (_, health) = self.configuration()?;
4251        let drain_timeout_ms = u64::try_from(
4252            self.inner
4253                .effective_drain_timeout
4254                .lock()
4255                .unwrap_or_else(|poisoned| poisoned.into_inner())
4256                .as_millis(),
4257        )
4258        .ok();
4259        self.update_configuration(spec, health, drain_timeout_ms)
4260            .await
4261    }
4262
4263    pub(crate) async fn update_configuration(
4264        &self,
4265        spec: ModuleSpec,
4266        health: HealthConfig,
4267        drain_timeout_ms: Option<u64>,
4268    ) -> Result<(), SuperviseError> {
4269        if spec.module_id != self.inner.module_id {
4270            return Err(SuperviseError::InvalidSpec {
4271                reason: "a supervised module's module_id cannot be changed".to_string(),
4272            });
4273        }
4274        validate_spec(&spec)?;
4275        let (reply_tx, reply_rx) = oneshot::channel();
4276        self.inner
4277            .commands
4278            .send(SupervisorCommand::UpdateConfiguration {
4279                spec: spec.clone(),
4280                health: health.clone(),
4281                drain_timeout_ms,
4282                reply: reply_tx,
4283            })
4284            .await
4285            .map_err(|_| SuperviseError::CommandClosed {
4286                module_id: self.inner.module_id.clone(),
4287            })?;
4288        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4289            module_id: self.inner.module_id.clone(),
4290        })?;
4291        let mut configuration =
4292            self.inner
4293                .configuration
4294                .lock()
4295                .map_err(|_| SuperviseError::StatePoisoned {
4296                    module_id: Some(self.inner.module_id.clone()),
4297                })?;
4298        configuration.spec = spec;
4299        configuration.health = health;
4300        Ok(())
4301    }
4302}
4303
4304impl Drop for SupervisedModuleInner {
4305    fn drop(&mut self) {
4306        let Ok(mut monitor) = self.monitor.lock() else {
4307            return;
4308        };
4309        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4310            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4311                state.state = ModuleState::Stopped;
4312                clear_current_process_facts(state);
4313            });
4314            monitor.abort();
4315        }
4316        let _ = monitor.take();
4317    }
4318}
4319
4320#[derive(Debug)]
4321enum SupervisorCommand {
4322    Drain {
4323        requester: Option<Principal>,
4324        reply: oneshot::Sender<Result<(), SuperviseError>>,
4325    },
4326    Retire {
4327        requester: Option<Principal>,
4328        reply: oneshot::Sender<Result<(), SuperviseError>>,
4329    },
4330    Restart {
4331        requester: Option<Principal>,
4332        /// Operator override for this one restart's drain budget, in ms. `None`
4333        /// uses the module's configured/default budget; `Some(0)` cuts
4334        /// immediately (wedge bounce: a stuck request never settles, so
4335        /// waiting only delays recovery).
4336        drain_timeout_ms: Option<u64>,
4337        /// The module's `spawn_generation` when the request was received, before
4338        /// it waited in the command queue. A queued restart whose module has
4339        /// since spawned a newer process is already satisfied (see the handler).
4340        received_at_generation: u64,
4341        /// When the request entered the command queue, so the handler can log
4342        /// how long it waited behind the loop's other work.
4343        queued_at: Instant,
4344        reply: oneshot::Sender<Result<(), SuperviseError>>,
4345    },
4346    Reload {
4347        requester: Option<Principal>,
4348        reply: oneshot::Sender<Result<(), SuperviseError>>,
4349    },
4350    SetEnabled {
4351        requester: Option<Principal>,
4352        enabled: bool,
4353        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4354    },
4355    UpdateConfiguration {
4356        spec: ModuleSpec,
4357        health: HealthConfig,
4358        /// Per-module drain override from the new config; `None` re-resolves to
4359        /// the supervisor-wide default.
4360        drain_timeout_ms: Option<u64>,
4361        reply: oneshot::Sender<()>,
4362    },
4363    Swap {
4364        requester: Option<Principal>,
4365        /// How long the candidate may take to register and declare itself
4366        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4367        ready_timeout: Option<Duration>,
4368        /// Answered at cutover or failure; the incumbent's drain follows.
4369        reply: oneshot::Sender<Result<(), SuperviseError>>,
4370    },
4371}
4372
4373#[derive(Debug)]
4374pub enum SuperviseError {
4375    InvalidSpec {
4376        reason: String,
4377    },
4378    Spawn {
4379        program: PathBuf,
4380        source: io::Error,
4381        cgroup_path: Option<PathBuf>,
4382    },
4383    Cgroup {
4384        module_id: String,
4385        source: io::Error,
4386    },
4387    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4388    /// than spawn a reserved module without its identity binding.
4389    LaunchNonce {
4390        reason: String,
4391    },
4392    Wait {
4393        module_id: String,
4394        source: io::Error,
4395    },
4396    Kill {
4397        module_id: String,
4398        source: io::Error,
4399    },
4400    Forwarding(ForwardingError),
4401    Registry(RegistryError),
4402    ReloadUnavailable {
4403        module_id: String,
4404        reason: String,
4405    },
4406    /// An operator restart/reload was requested for a module that is currently
4407    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4408    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4409    /// by a restart, so these commands are rejected instead of re-enabling it.
4410    Disabled {
4411        module_id: String,
4412    },
4413    ReloadFailed {
4414        module_id: String,
4415        reason: String,
4416    },
4417    RegistrationStillActive {
4418        module_id: String,
4419        waited: Duration,
4420    },
4421    StatePoisoned {
4422        module_id: Option<String>,
4423    },
4424    CommandClosed {
4425        module_id: String,
4426    },
4427    /// A restart or reload arrived while a swap's candidate was warming. The
4428    /// swap owns the module until it cuts over or fails; a stop or disable
4429    /// would have aborted it instead.
4430    SwapInProgress {
4431        module_id: String,
4432    },
4433    /// A swap was refused before anything was spawned.
4434    SwapRefused {
4435        module_id: String,
4436        reason: SwapRefusal,
4437    },
4438    /// A swap spawned a candidate and gave up on it. The candidate has been
4439    /// killed and its slot freed; the incumbent was left serving and was never
4440    /// drained, except in the one `CutoverLost` case described on that arm.
4441    SwapFailed {
4442        module_id: String,
4443        arm: SwapFailureArm,
4444        detail: String,
4445        /// How the candidate exited, when it exited on its own before the
4446        /// supervisor gave up on it.
4447        candidate_exit: Option<ExitReport>,
4448    },
4449}
4450
4451/// Why a swap was refused before a candidate was spawned.
4452#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4453pub enum SwapRefusal {
4454    /// The module's config does not declare `overlap: "safe"`.
4455    OverlapExclusive,
4456    /// The module is not registered, so there is no incumbent to keep serving
4457    /// and nothing a swap would improve on; a plain restart is the tool.
4458    NotRegistered,
4459    /// The module does not speak the subc wire, so a candidate could never
4460    /// register or declare itself ready.
4461    ProtocolNone,
4462    /// The supervisor lacks the forwarding table (to cut routes over) or the
4463    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4464    NotConfigured,
4465    /// A swap is already open for this module.
4466    AlreadySwapping,
4467}
4468
4469impl SwapRefusal {
4470    pub fn as_str(self) -> &'static str {
4471        match self {
4472            Self::OverlapExclusive => "overlap_exclusive",
4473            Self::NotRegistered => "not_registered",
4474            Self::ProtocolNone => "protocol_none",
4475            Self::NotConfigured => "not_configured",
4476            Self::AlreadySwapping => "already_swapping",
4477        }
4478    }
4479}
4480
4481/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4482/// serving and undrained; see `CutoverLost`.
4483#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4484pub enum SwapFailureArm {
4485    /// The candidate process could not be started.
4486    SpawnFailed,
4487    /// The candidate did not register within the readiness budget.
4488    NeverRegistered,
4489    /// The candidate registered but did not declare itself ready in time.
4490    NeverReady,
4491    /// The candidate exited before cutover.
4492    CandidateExited,
4493    /// The candidate declared itself ready but failed its health probe.
4494    CandidateUnhealthy,
4495    /// An operator stop, disable or retire arrived while the candidate warmed.
4496    /// The candidate was killed and the operator's command then carried out on
4497    /// the incumbent.
4498    Interrupted,
4499    /// The candidate's connection closed at the moment of cutover. If it
4500    /// closed before forwarding moved, the incumbent is untouched. If it closed
4501    /// between the forwarding and registry halves of cutover, forwarding can no
4502    /// longer route to the incumbent, so the module is restarted plainly.
4503    CutoverLost,
4504}
4505
4506impl SwapFailureArm {
4507    pub fn as_str(self) -> &'static str {
4508        match self {
4509            Self::SpawnFailed => "spawn_failed",
4510            Self::NeverRegistered => "never_registered",
4511            Self::NeverReady => "never_ready",
4512            Self::CandidateExited => "candidate_exited",
4513            Self::CandidateUnhealthy => "candidate_unhealthy",
4514            Self::Interrupted => "interrupted",
4515            Self::CutoverLost => "cutover_lost",
4516        }
4517    }
4518}
4519
4520impl fmt::Display for SuperviseError {
4521    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4522        match self {
4523            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4524            Self::Spawn {
4525                program,
4526                source,
4527                cgroup_path: Some(cgroup_path),
4528            } => write!(
4529                f,
4530                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4531                cgroup_path.display(),
4532                program.display()
4533            ),
4534            Self::Spawn {
4535                program,
4536                source,
4537                cgroup_path: None,
4538            } => write!(
4539                f,
4540                "failed to spawn module '{}': {source}",
4541                program.display()
4542            ),
4543            Self::Cgroup { module_id, source } => {
4544                write!(
4545                    f,
4546                    "failed to prepare cgroup for module '{module_id}': {source}"
4547                )
4548            }
4549            Self::LaunchNonce { reason } => {
4550                write!(
4551                    f,
4552                    "failed to generate reserved-module launch nonce: {reason}"
4553                )
4554            }
4555            Self::Wait { module_id, source } => {
4556                write!(f, "failed to wait for module '{module_id}': {source}")
4557            }
4558            Self::Kill { module_id, source } => {
4559                write!(f, "failed to kill module '{module_id}': {source}")
4560            }
4561            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4562            Self::Registry(err) => write!(f, "registry error: {err}"),
4563            Self::ReloadUnavailable { module_id, reason } => {
4564                write!(f, "reload unavailable for module '{module_id}': {reason}")
4565            }
4566            Self::Disabled { module_id } => {
4567                write!(
4568                    f,
4569                    "module '{module_id}' is disabled; enable it before restart or reload"
4570                )
4571            }
4572            Self::ReloadFailed { module_id, reason } => {
4573                write!(f, "reload failed for module '{module_id}': {reason}")
4574            }
4575            Self::RegistrationStillActive { module_id, waited } => write!(
4576                f,
4577                "module '{module_id}' registration remained active after waiting {waited:?}"
4578            ),
4579            Self::StatePoisoned { module_id } => match module_id {
4580                Some(module_id) => {
4581                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4582                }
4583                None => write!(f, "supervisor state was poisoned"),
4584            },
4585            Self::CommandClosed { module_id } => {
4586                write!(
4587                    f,
4588                    "supervisor command channel for module '{module_id}' is closed"
4589                )
4590            }
4591            Self::SwapInProgress { module_id } => write!(
4592                f,
4593                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4594            ),
4595            Self::SwapRefused { module_id, reason } => match reason {
4596                SwapRefusal::OverlapExclusive => write!(
4597                    f,
4598                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4599                ),
4600                SwapRefusal::NotRegistered => write!(
4601                    f,
4602                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4603                ),
4604                SwapRefusal::ProtocolNone => write!(
4605                    f,
4606                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4607                ),
4608                SwapRefusal::NotConfigured => write!(
4609                    f,
4610                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4611                ),
4612                SwapRefusal::AlreadySwapping => {
4613                    write!(f, "module '{module_id}' is already being swapped")
4614                }
4615            },
4616            Self::SwapFailed {
4617                module_id,
4618                arm,
4619                detail,
4620                ..
4621            } => write!(
4622                f,
4623                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4624                arm.as_str()
4625            ),
4626        }
4627    }
4628}
4629
4630impl Error for SuperviseError {
4631    fn source(&self) -> Option<&(dyn Error + 'static)> {
4632        match self {
4633            Self::Spawn { source, .. }
4634            | Self::Cgroup { source, .. }
4635            | Self::Wait { source, .. }
4636            | Self::Kill { source, .. } => Some(source),
4637            Self::Forwarding(err) => Some(err),
4638            Self::Registry(err) => Some(err),
4639            Self::LaunchNonce { .. }
4640            | Self::InvalidSpec { .. }
4641            | Self::ReloadUnavailable { .. }
4642            | Self::Disabled { .. }
4643            | Self::ReloadFailed { .. }
4644            | Self::RegistrationStillActive { .. }
4645            | Self::StatePoisoned { .. }
4646            | Self::CommandClosed { .. }
4647            | Self::SwapInProgress { .. }
4648            | Self::SwapRefused { .. }
4649            | Self::SwapFailed { .. } => None,
4650        }
4651    }
4652}
4653
4654pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4655    if spec.module_id.trim().is_empty() {
4656        return Err(SuperviseError::InvalidSpec {
4657            reason: "module_id must not be empty".to_string(),
4658        });
4659    }
4660
4661    Ok(())
4662}
4663
4664#[derive(Debug, Default)]
4665struct HealthProbeRuntime {
4666    spawn_generation: Option<u64>,
4667    configured_health: Option<HealthConfig>,
4668    registered_connection: Option<crate::ConnectionId>,
4669    advertised: bool,
4670    next_probe_at: Option<Instant>,
4671    probe_index: u64,
4672}
4673
4674fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4675    lock_snapshot(snapshot)
4676        .ok()
4677        .and_then(|state| state.spawned_protocol)
4678        .unwrap_or(spec.protocol)
4679}
4680
4681impl HealthProbeRuntime {
4682    fn refresh_registration(
4683        &mut self,
4684        spec: &ModuleSpec,
4685        runtime: &SupervisorRuntimeConfig,
4686        registry: &Registry,
4687        snapshot: &SharedSnapshot,
4688    ) {
4689        let generation = lock_snapshot(snapshot)
4690            .ok()
4691            .map(|state| state.spawn_generation);
4692        if self.spawn_generation != generation {
4693            // A replacement cannot reuse a deadline that is already due for its
4694            // predecessor. HTTP children get a full cadence to start; wire
4695            // children still wait for their own health-advertising HELLO.
4696            *self = Self {
4697                spawn_generation: generation,
4698                ..Self::default()
4699            };
4700        }
4701        if self.configured_health.as_ref() != Some(&runtime.health) {
4702            self.configured_health = Some(runtime.health.clone());
4703            self.next_probe_at = None;
4704            self.registered_connection = None;
4705            self.probe_index = 0;
4706        }
4707        // A non-wire process never registers. Only an explicitly configured
4708        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4709        // health failure for that kind of process.
4710        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4711            self.registered_connection = None;
4712            self.advertised = runtime.health.http.is_some();
4713            if !self.advertised {
4714                self.next_probe_at = None;
4715                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4716                    let unknown = ModuleHealthStatus::default();
4717                    if state.health != unknown {
4718                        state.health = unknown;
4719                    }
4720                });
4721            } else if self.next_probe_at.is_none() {
4722                self.next_probe_at = Some(
4723                    Instant::now()
4724                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4725                );
4726            }
4727            return;
4728        }
4729
4730        let registration = match registry.get_module(&spec.module_id) {
4731            Ok(registration) => registration,
4732            Err(err) => {
4733                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4734                self.advertised = false;
4735                self.next_probe_at = None;
4736                return;
4737            }
4738        };
4739
4740        let Some(registration) = registration else {
4741            self.registered_connection = None;
4742            self.advertised = false;
4743            self.next_probe_at = None;
4744            return;
4745        };
4746
4747        let advertised = registration
4748            .control_ops
4749            .iter()
4750            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4751        if !advertised {
4752            self.registered_connection = Some(registration.connection_id);
4753            self.advertised = false;
4754            self.next_probe_at = None;
4755            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4756                if state.health.status != SupervisorHealthStatus::Unknown
4757                    || state.health.consecutive_failures != 0
4758                    || state.health.last_probe_ms.is_some()
4759                    || state.health.detail.is_some()
4760                    || state.health.metrics.is_some()
4761                {
4762                    state.health.status = SupervisorHealthStatus::Unknown;
4763                    state.health.consecutive_failures = 0;
4764                    state.health.last_probe_ms = None;
4765                    state.health.detail = None;
4766                    state.health.metrics = None;
4767                }
4768            });
4769            return;
4770        }
4771
4772        let reregistered = self.registered_connection != Some(registration.connection_id);
4773        self.registered_connection = Some(registration.connection_id);
4774        self.advertised = true;
4775        if reregistered || self.next_probe_at.is_none() {
4776            self.probe_index = 0;
4777            self.next_probe_at = Some(
4778                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4779            );
4780            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4781                state.health.status = SupervisorHealthStatus::Unknown;
4782                state.health.consecutive_failures = 0;
4783                state.health.detail = None;
4784                state.health.metrics = None;
4785            });
4786        }
4787    }
4788
4789    fn wake_at(&self) -> Option<Instant> {
4790        if !self.advertised {
4791            return None;
4792        }
4793        self.next_probe_at
4794    }
4795
4796    fn due(&self) -> bool {
4797        self.advertised
4798            && self
4799                .next_probe_at
4800                .is_some_and(|next| Instant::now() >= next)
4801    }
4802
4803    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4804        self.probe_index = self.probe_index.wrapping_add(1);
4805        self.next_probe_at = Some(
4806            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4807        );
4808    }
4809}
4810
4811/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4812///
4813/// This was a struct with a single `message: String`, and every one of the
4814/// fifteen construction sites collapsed into it. Each site knows exactly what it
4815/// saw -- the lane is gone, the module did not answer in time, the module
4816/// answered with the wrong thing -- and `handle_health_probe_failure` then
4817/// treated all of them identically: increment a counter, compare to a threshold,
4818/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4819/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4820///
4821/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4822///
4823/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4824///   answer on it again.
4825/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4826///   AND with a perfectly healthy one that lost a CPU race -- which is what
4827///   happens under machine load, and is how this supervisor killed a healthy
4828///   module three times in one day.
4829/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4830///   Restarting on it is defensible, but it is not the silence case and should
4831///   never be counted as one.
4832/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4833///   anything, so it cannot be evidence about the module at all.
4834///
4835/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4836/// one that fires most often, and while every variant collapsed into one string
4837/// it carried the same weight as the strongest.
4838///
4839/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4840/// DESIGN and a reader stopping at it gets the build backwards: the restart
4841/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4842/// probes still increment the failure streak and drive escalation at the
4843/// threshold (see `is_proof_of_death` below for why that is deliberate and
4844/// what gates the change). Absence of evidence restarts modules today.
4845#[derive(Debug)]
4846enum HealthProbeEvidence {
4847    /// The module's control lane is gone. Proof of death.
4848    LaneDead,
4849    /// No reply within the deadline. Proves nothing about the module's state.
4850    NoAnswer,
4851    /// The module replied, but not with a usable health report. Proves it is alive.
4852    BadAnswer,
4853    /// The daemon could not ask. Says nothing about the module.
4854    Misconfigured,
4855}
4856
4857#[derive(Debug)]
4858struct HealthProbeError {
4859    evidence: HealthProbeEvidence,
4860    message: String,
4861}
4862
4863impl HealthProbeError {
4864    fn lane_dead(message: impl Into<String>) -> Self {
4865        Self::with(HealthProbeEvidence::LaneDead, message)
4866    }
4867
4868    fn no_answer(message: impl Into<String>) -> Self {
4869        Self::with(HealthProbeEvidence::NoAnswer, message)
4870    }
4871
4872    fn bad_answer(message: impl Into<String>) -> Self {
4873        Self::with(HealthProbeEvidence::BadAnswer, message)
4874    }
4875
4876    fn misconfigured(message: impl Into<String>) -> Self {
4877        Self::with(HealthProbeEvidence::Misconfigured, message)
4878    }
4879
4880    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4881        Self {
4882            evidence,
4883            message: message.into(),
4884        }
4885    }
4886
4887    /// Whether this observation is proof the module cannot serve.
4888    ///
4889    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4890    /// variant that fires under CPU starvation, and treating it as proof is the
4891    /// defect this enum exists to make impossible to reintroduce silently.
4892    ///
4893    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4894    /// to restart also needs a bound for the case it excludes -- a genuinely
4895    /// wedged module, alive but never answering -- and that bound must come from
4896    /// the distribution of real late-answer latencies, which nothing measures
4897    /// yet. Landing the classification first makes the later change a one-line
4898    /// decision against evidence that already exists, rather than two unproven
4899    /// changes at once.
4900    #[allow(dead_code)]
4901    fn is_proof_of_death(&self) -> bool {
4902        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4903    }
4904
4905    /// Short stable label for logs and the health snapshot.
4906    ///
4907    /// An operator reading `ck health` currently cannot tell "the module is gone"
4908    /// from "the module did not answer in five seconds", because both render as
4909    /// prose in the same field. These labels are what make the two
4910    /// distinguishable at a glance, and they are what a later restart-policy
4911    /// change will be argued from.
4912    fn label(&self) -> &'static str {
4913        match self.evidence {
4914            HealthProbeEvidence::LaneDead => "lane-dead",
4915            HealthProbeEvidence::NoAnswer => "no-answer",
4916            HealthProbeEvidence::BadAnswer => "bad-answer",
4917            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4918        }
4919    }
4920}
4921
4922impl fmt::Display for HealthProbeError {
4923    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4924        f.write_str(&self.message)
4925    }
4926}
4927
4928async fn run_health_probe_cycle(
4929    spec: &ModuleSpec,
4930    runtime: &SupervisorRuntimeConfig,
4931    registry: &Registry,
4932    process_liveness: &SupervisorProcessLiveness,
4933    snapshot: &SharedSnapshot,
4934    child: &mut Option<SupervisedChild>,
4935) {
4936    let now_ms = unix_ms_now();
4937    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4938        .then_some(runtime.health.http.as_deref())
4939        .flatten();
4940    let result = match http {
4941        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4942        None => probe_module_health(&spec.module_id, runtime, None).await,
4943    };
4944    match result {
4945        Ok(report) => {
4946            handle_health_report(
4947                spec,
4948                runtime,
4949                registry,
4950                process_liveness,
4951                snapshot,
4952                child,
4953                report,
4954                now_ms,
4955            )
4956            .await;
4957        }
4958        Err(err) => {
4959            if http.is_some() {
4960                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4961                    state.health.status = SupervisorHealthStatus::Failing;
4962                });
4963            }
4964            handle_health_probe_failure(
4965                spec,
4966                runtime,
4967                registry,
4968                process_liveness,
4969                snapshot,
4970                child,
4971                err,
4972                now_ms,
4973            )
4974            .await;
4975        }
4976    }
4977}
4978
4979pub(crate) struct HttpProbeTarget<'a> {
4980    address: std::net::SocketAddr,
4981    localhost: bool,
4982    authority: &'a str,
4983    path: String,
4984}
4985
4986/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4987/// or TLS. A URL cannot turn a local health check into an outbound connection.
4988pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4989    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4990        return Err("must not contain whitespace, controls, or a fragment".into());
4991    }
4992    let rest = url
4993        .strip_prefix("http://")
4994        .ok_or("must use plain http://")?;
4995    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4996    let (authority, suffix) = rest.split_at(split);
4997    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4998        ("::1", rest)
4999    } else {
5000        let split = authority.find(':').unwrap_or(authority.len());
5001        authority.split_at(split)
5002    };
5003    let ip: std::net::IpAddr = match host {
5004        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5005        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5006        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5007    };
5008    let port = if port.is_empty() {
5009        80
5010    } else {
5011        port.strip_prefix(':')
5012            .and_then(|p| p.parse::<u16>().ok())
5013            .filter(|p| *p > 0)
5014            .ok_or("must have a valid nonzero TCP port")?
5015    };
5016    let path = if suffix.is_empty() {
5017        "/".into()
5018    } else if suffix.starts_with('?') {
5019        format!("/{suffix}")
5020    } else {
5021        suffix.into()
5022    };
5023    Ok(HttpProbeTarget {
5024        address: std::net::SocketAddr::new(ip, port),
5025        localhost: host == "localhost",
5026        authority,
5027        path,
5028    })
5029}
5030
5031async fn probe_http_health(
5032    url: &str,
5033    deadline: Duration,
5034) -> Result<HealthReport, HealthProbeError> {
5035    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5036    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5037    // Keep partial diagnostics outside the timed future so cancellation does
5038    // not discard a status line or body bytes already received.
5039    let mut response_status = String::new();
5040    let mut body = Vec::new();
5041    let probe = async {
5042        // Resolve localhost ourselves so a hosts-file override cannot turn
5043        // this into an outbound request, while IPv6-only local servers work.
5044        let connection = match tokio::net::TcpStream::connect(target.address).await {
5045            Err(_) if target.localhost => {
5046                tokio::net::TcpStream::connect((
5047                    std::net::Ipv6Addr::LOCALHOST,
5048                    target.address.port(),
5049                ))
5050                .await
5051            }
5052            result => result,
5053        };
5054        let mut stream = connection.map_err(|error| {
5055            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5056        })?;
5057        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5058            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5059        let mut reader = BufReader::new(stream);
5060        let mut budget = 16 * 1024;
5061        let status = http_line(&mut reader, &mut budget).await?;
5062        let mut words = status.split_ascii_whitespace();
5063        let version = words.next();
5064        let code = words
5065            .next()
5066            .filter(|word| word.len() == 3)
5067            .and_then(|word| word.parse::<u16>().ok());
5068        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5069            || !code.is_some_and(|code| (100..600).contains(&code))
5070        {
5071            return Err(HealthProbeError::bad_answer(format!(
5072                "invalid HTTP status: {status}"
5073            )));
5074        }
5075        let code = code.expect("validated status code");
5076        response_status = status.clone();
5077        let mut length = None;
5078        let mut chunked = false;
5079        loop {
5080            let line = http_line(&mut reader, &mut budget).await?;
5081            if line.is_empty() {
5082                break;
5083            }
5084            if let Some((name, value)) = line.split_once(':') {
5085                if name.eq_ignore_ascii_case("content-length") {
5086                    length = Some(value.trim().parse::<u64>().map_err(|_| {
5087                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
5088                    })?);
5089                } else if name.eq_ignore_ascii_case("transfer-encoding") {
5090                    chunked = value.trim().eq_ignore_ascii_case("chunked");
5091                }
5092            }
5093        }
5094        if chunked {
5095            while body.len() < 200 {
5096                let line = http_line(&mut reader, &mut budget).await?;
5097                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5098                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5099                if size == 0 {
5100                    break;
5101                }
5102                let count = size.min((200 - body.len()) as u64) as usize;
5103                let start = body.len();
5104                (&mut reader)
5105                    .take(count as u64)
5106                    .read_to_end(&mut body)
5107                    .await
5108                    .map_err(|error| {
5109                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5110                    })?;
5111                if body.len() - start != count {
5112                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5113                }
5114                if size > count as u64 || body.len() == 200 {
5115                    break;
5116                }
5117                if !http_line(&mut reader, &mut budget).await?.is_empty() {
5118                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5119                }
5120            }
5121        } else {
5122            reader
5123                .take(length.unwrap_or(200).min(200))
5124                .read_to_end(&mut body)
5125                .await
5126                .map_err(|error| {
5127                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5128                })?;
5129        }
5130        if (200..300).contains(&code) {
5131            Ok(HealthReport::ok())
5132        } else {
5133            Err(HealthProbeError::bad_answer(
5134                "HTTP health endpoint returned non-2xx",
5135            ))
5136        }
5137    };
5138    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5139        Err(HealthProbeError::no_answer(format!(
5140            "HTTP probe timed out after {deadline:?}"
5141        )))
5142    });
5143    if let Err(error) = &mut result {
5144        if !response_status.is_empty() {
5145            error.message = format!(
5146                "{}; {response_status}: {}",
5147                error.message,
5148                String::from_utf8_lossy(&body)
5149            );
5150        }
5151    }
5152    result
5153}
5154
5155async fn http_line(
5156    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5157    remaining: &mut usize,
5158) -> Result<String, HealthProbeError> {
5159    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5160    let mut line = Vec::new();
5161    (&mut *reader)
5162        .take(*remaining as u64)
5163        .read_until(b'\n', &mut line)
5164        .await
5165        .map_err(|error| {
5166            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5167        })?;
5168    *remaining -= line.len();
5169    if !line.ends_with(b"\r\n") {
5170        return Err(HealthProbeError::bad_answer(
5171            "HTTP headers are incomplete or exceed 16 KiB",
5172        ));
5173    }
5174    line.truncate(line.len() - 2);
5175    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5176}
5177
5178async fn probe_module_health(
5179    module_id: &str,
5180    runtime: &SupervisorRuntimeConfig,
5181    drain_deadline: Option<Instant>,
5182) -> Result<HealthReport, HealthProbeError> {
5183    let Some(forwarding) = runtime.forwarding.as_ref() else {
5184        return Err(HealthProbeError::misconfigured(
5185            "supervisor was not configured with a forwarding table",
5186        ));
5187    };
5188    let probe_started_at = Instant::now();
5189    let mut deadline = probe_started_at + runtime.health.deadline;
5190    if let Some(drain_deadline) = drain_deadline {
5191        deadline = deadline.min(drain_deadline);
5192    }
5193    let pending = if drain_deadline.is_some() {
5194        forwarding.begin_drain_health_probe_rpc_for(
5195            module_id,
5196            MODULE_CONTROL_OP_HEALTH_CHECK,
5197            probe_started_at,
5198            deadline,
5199        )
5200    } else {
5201        forwarding.begin_health_probe_rpc_for(
5202            module_id,
5203            MODULE_CONTROL_OP_HEALTH_CHECK,
5204            probe_started_at,
5205            deadline,
5206        )
5207    }
5208    .map_err(|err| {
5209        // The endpoint is not registered, so there is no live control lane to
5210        // ask. That is the module being absent, not slow.
5211        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5212    })?;
5213    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5214}
5215
5216/// [`probe_module_health`] for one endpoint rather than the id's active one.
5217///
5218/// A swap probes two processes that no by-id lookup reaches: its candidate
5219/// before cutover, and its superseded incumbent (for busy gauges) while the
5220/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5221/// bounds the by-id drain probe.
5222async fn probe_endpoint_health(
5223    endpoint: crate::ModuleEndpointId,
5224    runtime: &SupervisorRuntimeConfig,
5225    deadline_cap: Option<Instant>,
5226) -> Result<HealthReport, HealthProbeError> {
5227    let Some(forwarding) = runtime.forwarding.as_ref() else {
5228        return Err(HealthProbeError::misconfigured(
5229            "supervisor was not configured with a forwarding table",
5230        ));
5231    };
5232    let probe_started_at = Instant::now();
5233    let mut deadline = probe_started_at + runtime.health.deadline;
5234    if let Some(cap) = deadline_cap {
5235        deadline = deadline.min(cap);
5236    }
5237    let pending = forwarding
5238        .begin_endpoint_health_probe_rpc_for(
5239            endpoint,
5240            MODULE_CONTROL_OP_HEALTH_CHECK,
5241            probe_started_at,
5242            deadline,
5243        )
5244        .map_err(|err| {
5245            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5246        })?;
5247    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5248}
5249
5250/// Send a begun health probe and classify its answer.
5251async fn await_health_probe(
5252    forwarding: &ForwardingTable,
5253    pending: PendingModuleControlRpc,
5254    deadline: Instant,
5255    probe_budget: Duration,
5256) -> Result<HealthReport, HealthProbeError> {
5257    let PendingModuleControlRpc {
5258        endpoint,
5259        module_sink,
5260        negotiated_ver,
5261        corr,
5262        receiver,
5263    } = pending;
5264    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5265        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5266    })?;
5267    let frame = Frame::build_with_version(
5268        negotiated_ver,
5269        FrameType::Request,
5270        control_flags(),
5271        0,
5272        0,
5273        corr,
5274        body,
5275    )
5276    .map_err(|err| {
5277        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5278    })?;
5279
5280    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5281    // blocks waiting for capacity when the module's egress queue is full, and an
5282    // unbounded await here freezes the whole supervision actor (it stops polling
5283    // Child::wait and supervisor commands), making the module unrecoverable
5284    // in-band. On timeout the probe fails like any transport failure.
5285    match timeout_at(deadline, module_sink.send(frame)).await {
5286        Ok(Ok(())) => {}
5287        Ok(Err(err)) => {
5288            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5289            // A closed sink means the module's egress channel is gone -- the
5290            // receiving half is dropped when its connection tears down. Proof.
5291            return Err(HealthProbeError::lane_dead(format!(
5292                "failed to send health.check: {err}"
5293            )));
5294        }
5295        Err(_elapsed) => {
5296            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5297            // A full egress queue means the module is not draining its socket, which
5298            // is consistent with a wedged module AND with one whose reader is merely
5299            // starved. Silence, not proof.
5300            return Err(HealthProbeError::no_answer(
5301                "health.check send timed out before enqueue (module egress full)",
5302            ));
5303        }
5304    }
5305
5306    match timeout_at(deadline, receiver).await {
5307        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5308        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5309        // and those prove it is alive even though the probe failed.
5310        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5311            response.health_report().ok_or_else(|| {
5312                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5313            })
5314        }
5315        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5316            format!("health.check rejected: {}", body.message),
5317        )),
5318        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5319            Err(HealthProbeError::lane_dead(message))
5320        }
5321        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5322            Err(HealthProbeError::bad_answer(message))
5323        }
5324        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5325            Err(HealthProbeError::bad_answer(format!(
5326                "expected module-control op '{expected}', got '{actual}'"
5327            )))
5328        }
5329        // A reply that crosses the deadline before this waiter observes it is
5330        // still proof of life. The forwarding path records its end-to-end latency
5331        // before delivering this classification.
5332        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5333            "module answered health.check after its daemon deadline",
5334        )),
5335        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5336            "health.check waiter was canceled before the module responded",
5337        )),
5338        Err(_) => {
5339            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5340            Err(HealthProbeError::no_answer(format!(
5341                "module did not answer health.check within {probe_budget:?}"
5342            )))
5343        }
5344    }
5345}
5346
5347#[allow(clippy::too_many_arguments)]
5348async fn handle_health_report(
5349    spec: &ModuleSpec,
5350    runtime: &SupervisorRuntimeConfig,
5351    registry: &Registry,
5352    process_liveness: &SupervisorProcessLiveness,
5353    snapshot: &SharedSnapshot,
5354    child: &mut Option<SupervisedChild>,
5355    report: HealthReport,
5356    now_ms: u64,
5357) {
5358    let status = supervisor_health_status(report.status);
5359    let detail = report.detail.clone();
5360    let metrics = truncate_health_metrics(report.metrics);
5361    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5362        state.health.status = status;
5363        state.health.last_probe_ms = Some(now_ms);
5364        state.health.detail = detail.clone();
5365        state.health.metrics = metrics.clone();
5366        state.health.consecutive_failures = 0;
5367        state.failed_retry.observe_health(
5368            report.status == HealthStatus::Ok,
5369            runtime.restart_policy.window,
5370        );
5371    });
5372
5373    let action = match report.status {
5374        HealthStatus::Ok => return,
5375        HealthStatus::Degraded => runtime.health.on_degraded,
5376        HealthStatus::Failing => runtime.health.on_failing,
5377    };
5378    apply_l3_health_action(
5379        spec,
5380        runtime,
5381        registry,
5382        process_liveness,
5383        snapshot,
5384        child,
5385        status,
5386        detail.as_deref(),
5387        action,
5388        now_ms,
5389    )
5390    .await;
5391}
5392
5393#[allow(clippy::too_many_arguments)]
5394async fn handle_health_probe_failure(
5395    spec: &ModuleSpec,
5396    runtime: &SupervisorRuntimeConfig,
5397    registry: &Registry,
5398    process_liveness: &SupervisorProcessLiveness,
5399    snapshot: &SharedSnapshot,
5400    child: &mut Option<SupervisedChild>,
5401    err: HealthProbeError,
5402    now_ms: u64,
5403) {
5404    let threshold = runtime.health.failure_threshold.max(1);
5405    let mut failures = 0;
5406    // Carry the evidence class into the operator-visible detail. Without it,
5407    // "module did not answer within 5s" and "the control lane is gone" are two
5408    // prose strings in the same field, and the reader has to know the codebase to
5409    // tell which one is proof of anything.
5410    let detail = format!("[{}] {err}", err.label());
5411    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5412        state
5413            .failed_retry
5414            .observe_health(false, runtime.restart_policy.window);
5415        // A failed wire probe invalidates the last report, even before the
5416        // restart threshold. HTTP probes already mark failures as Failing.
5417        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5418            state.health.status = SupervisorHealthStatus::Unknown;
5419        }
5420        state.health.last_probe_ms = Some(now_ms);
5421        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5422        state.health.detail = Some(detail.clone());
5423        state.health.metrics = None;
5424        failures = state.health.consecutive_failures;
5425    });
5426
5427    if failures < threshold {
5428        warn!(
5429            module_id = %spec.module_id,
5430            consecutive_failures = failures,
5431            threshold,
5432            evidence = err.label(),
5433            detail = %detail,
5434            "health.check probe failed"
5435        );
5436        return;
5437    }
5438
5439    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5440        state.state = ModuleState::Unresponsive;
5441        state.health.status = SupervisorHealthStatus::Unresponsive;
5442    });
5443    // The evidence class is logged at the kill site because this is the line an
5444    // operator reads after an unexplained restart. A streak of `no-answer` under
5445    // machine load is the known false-positive shape; a `lane-dead` is not.
5446    if runtime.health.critical {
5447        error!(
5448            module_id = %spec.module_id,
5449            status = "unresponsive",
5450            evidence = err.label(),
5451            detail = %detail,
5452            "critical module health alert"
5453        );
5454    } else {
5455        warn!(
5456            module_id = %spec.module_id,
5457            status = "unresponsive",
5458            evidence = err.label(),
5459            detail = %detail,
5460            "module health threshold breached"
5461        );
5462    }
5463    if let Err(err) = health_restart_child(
5464        spec,
5465        runtime,
5466        registry,
5467        process_liveness,
5468        snapshot,
5469        child,
5470        SupervisorHealthStatus::Unresponsive,
5471        Some(&detail),
5472        now_ms,
5473    )
5474    .await
5475    {
5476        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5477    }
5478}
5479
5480#[allow(clippy::too_many_arguments)]
5481async fn apply_l3_health_action(
5482    spec: &ModuleSpec,
5483    runtime: &SupervisorRuntimeConfig,
5484    registry: &Registry,
5485    process_liveness: &SupervisorProcessLiveness,
5486    snapshot: &SharedSnapshot,
5487    child: &mut Option<SupervisedChild>,
5488    status: SupervisorHealthStatus,
5489    detail: Option<&str>,
5490    action: HealthAction,
5491    now_ms: u64,
5492) {
5493    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5494    match action {
5495        HealthAction::Report => {
5496            info!(
5497                module_id = %spec.module_id,
5498                status = ?status,
5499                detail,
5500                "module reported non-ok health"
5501            );
5502        }
5503        HealthAction::Alert => {
5504            error!(
5505                module_id = %spec.module_id,
5506                status = ?status,
5507                detail,
5508                "module health alert"
5509            );
5510        }
5511        HealthAction::Restart => {
5512            if let Err(err) = health_restart_child(
5513                spec,
5514                runtime,
5515                registry,
5516                process_liveness,
5517                snapshot,
5518                child,
5519                status,
5520                detail,
5521                now_ms,
5522            )
5523            .await
5524            {
5525                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5526            }
5527        }
5528    }
5529}
5530
5531#[allow(clippy::too_many_arguments)]
5532async fn health_restart_child(
5533    spec: &ModuleSpec,
5534    runtime: &SupervisorRuntimeConfig,
5535    registry: &Registry,
5536    process_liveness: &SupervisorProcessLiveness,
5537    snapshot: &SharedSnapshot,
5538    child: &mut Option<SupervisedChild>,
5539    status: SupervisorHealthStatus,
5540    detail: Option<&str>,
5541    now_ms: u64,
5542) -> Result<(), SuperviseError> {
5543    let (enabled, schedule) = {
5544        let mut state = lock_snapshot(snapshot)?;
5545        let enabled = state.enabled;
5546        let schedule = if enabled {
5547            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5548        } else {
5549            None
5550        };
5551        (enabled, schedule)
5552    };
5553
5554    if !enabled {
5555        return Err(SuperviseError::Disabled {
5556            module_id: spec.module_id.clone(),
5557        });
5558    }
5559
5560    if schedule.is_none() {
5561        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5562        error!(
5563            module_id = %spec.module_id,
5564            status = ?status,
5565            detail,
5566            max_restarts = runtime.restart_policy.max_restarts,
5567            window_secs = runtime.restart_policy.window.as_secs(),
5568            reason = %runtime.restart_policy.budget_exhausted_detail(),
5569            "health restart budget exhausted; marking module failed"
5570        );
5571        let stop_notice = begin_forwarding_drain_if_configured(
5572            spec,
5573            runtime,
5574            registry,
5575            snapshot,
5576            Some(true),
5577            RouteCloseReason::Disable,
5578        )
5579        .await?;
5580        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5581            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5582        })?;
5583        let result = drain_optional_child(
5584            &spec.module_id,
5585            spec.protocol,
5586            stop_notice,
5587            registry,
5588            runtime.forwarding.as_deref(),
5589            snapshot,
5590            &runtime.terminal_ring,
5591            &runtime.spawn_events,
5592            child,
5593            runtime.drain_timeout,
5594            ModuleState::Failed,
5595            Some(true),
5596            &runtime.failure_context(true),
5597        )
5598        .await;
5599        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5600            if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5601                state.failed_retry.schedule(&spec.module_id);
5602            }
5603        })?;
5604        result?;
5605        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5606        return Ok(());
5607    }
5608
5609    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5610    let mut restart_count = 0;
5611    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5612        restart_count = state.crash_restarts.len();
5613        state.state = ModuleState::Unresponsive;
5614        state.health.status = status;
5615        state.health.last_action = Some(HealthAction::Restart.to_string());
5616        state.health.last_action_ms = Some(now_ms);
5617    })?;
5618    warn!(
5619        module_id = %spec.module_id,
5620        status = ?status,
5621        detail,
5622        restart_count,
5623        restart_in_window = schedule.restart_in_window,
5624        delay_ms = schedule.delay.as_millis() as u64,
5625        "health-triggered module restart"
5626    );
5627
5628    let stop_notice = begin_forwarding_drain_if_configured(
5629        spec,
5630        runtime,
5631        registry,
5632        snapshot,
5633        Some(true),
5634        RouteCloseReason::Restart,
5635    )
5636    .await?;
5637    drain_optional_child(
5638        &spec.module_id,
5639        spec.protocol,
5640        stop_notice,
5641        registry,
5642        runtime.forwarding.as_deref(),
5643        snapshot,
5644        &runtime.terminal_ring,
5645        &runtime.spawn_events,
5646        child,
5647        runtime.drain_timeout,
5648        ModuleState::Restarting,
5649        Some(true),
5650        &runtime.failure_context(true),
5651    )
5652    .await?;
5653    schedule_respawn(
5654        runtime,
5655        snapshot,
5656        &spec.module_id,
5657        schedule.delay,
5658        RespawnKind::Spawn,
5659        true,
5660    )
5661}
5662
5663fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5664    if let Some(reply) = runtime
5665        .deferred_reload_reply
5666        .lock()
5667        .unwrap_or_else(|p| p.into_inner())
5668        .take()
5669    {
5670        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5671            module_id: module_id.to_string(),
5672            reason: reason.to_string(),
5673        }));
5674    }
5675}
5676
5677fn schedule_respawn(
5678    runtime: &SupervisorRuntimeConfig,
5679    snapshot: &SharedSnapshot,
5680    module_id: &str,
5681    delay: Duration,
5682    kind: RespawnKind,
5683    daemon_initiated: bool,
5684) -> Result<(), SuperviseError> {
5685    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5686    update_snapshot(snapshot, Some(module_id), |state| {
5687        state.respawn_pending = true
5688    })?;
5689    *runtime
5690        .scheduled_respawn
5691        .lock()
5692        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5693        deadline: Instant::now() + delay,
5694        kind,
5695        daemon_initiated,
5696    });
5697    Ok(())
5698}
5699
5700fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5701    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5702        state.health.last_action = Some(action);
5703        state.health.last_action_ms = Some(now_ms);
5704    });
5705}
5706
5707fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5708    match status {
5709        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5710        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5711        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5712    }
5713}
5714
5715/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5716/// returned to every `supervisor.list` and `supervisor.health` caller.
5717///
5718/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5719/// path: that request exists to return a module's complete metrics object, and
5720/// `ck health <module-id>` documents it as the way to see what the cached view
5721/// truncates. The asymmetry is the feature.
5722///
5723/// So a new caller must decide which side it is on rather than assume the cap is
5724/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5725/// the truncation that path exists to avoid.
5726fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5727    let metrics = metrics?;
5728    match serde_json::to_vec(&metrics) {
5729        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5730            "truncated": true,
5731            "original_bytes": encoded.len(),
5732        })),
5733        Ok(_) | Err(_) => Some(metrics),
5734    }
5735}
5736
5737/// Spread health probes so a fleet-wide restart does not converge them.
5738///
5739/// The delay is derived from the module id and probe index rather than a random
5740/// source, so it is deterministic per module: a module keeps its own offset
5741/// across daemon restarts instead of re-rolling into a collision.
5742fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5743    if cadence.is_zero() {
5744        return Duration::ZERO;
5745    }
5746    let cadence_ms = cadence.as_millis() as u64;
5747    // This early return is REDUNDANT, deliberately, and a mutation run will show
5748    // it surviving removal. Recording why here so the next person to notice does
5749    // not have to re-derive it:
5750    //
5751    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5752    //   a zero cadence and builds the Duration from whole milliseconds, so a
5753    //   sub-millisecond cadence cannot come from config.
5754    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5755    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5756    //   -- exactly what this returns.
5757    //
5758    // Kept as a guard against a future widening of the config parser (accepting
5759    // microseconds, say), which would make the sub-millisecond case reachable.
5760    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5761    // divides by zero. Remove this and nothing changes.
5762    if cadence_ms == 0 {
5763        return cadence;
5764    }
5765    // Note that this never returns less than one cadence, including for the FIRST
5766    // probe. So a freshly registered module reports health `unknown` for a full
5767    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5768    // ready to answer.
5769    //
5770    // That is a property of the supervisor's schedule, not of any module: an
5771    // operator watching a restart sees `unknown` and cannot tell it from a module
5772    // that is slow to warm. Measured on two unrelated modules, both flipping to
5773    // `ok` between 22s and 32s after restart.
5774    //
5775    // Left as-is because spreading the first probe is what keeps a fleet-wide
5776    // restart from firing fourteen simultaneous probes into a cold machine. The
5777    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5778    // that thundering herd for a faster first reading.
5779    let jitter_span = (cadence_ms / 10).max(1);
5780    let hash = module_id.as_bytes().iter().fold(
5781        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5782        |acc, byte| {
5783            acc.wrapping_mul(1099511628211)
5784                .wrapping_add(u64::from(*byte))
5785        },
5786    );
5787    cadence + Duration::from_millis(hash % jitter_span)
5788}
5789
5790#[cfg(test)]
5791mod tests {
5792    use super::*;
5793
5794    #[test]
5795    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5796        let handle = SupervisorHandle::new();
5797        let module_id = "readded-tombstone";
5798        handle.record_rescan_removal(module_id);
5799        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5800
5801        handle.apply_identity_configuration(&ModuleSpec {
5802            module_id: module_id.to_string(),
5803            program: PathBuf::from("/test/module"),
5804            args: Vec::new(),
5805            env: Vec::new(),
5806            reserved: false,
5807            reserved_prefixes: Vec::new(),
5808            protocol: ModuleProtocol::Subc,
5809            overlap: Default::default(),
5810        });
5811
5812        assert!(
5813            handle.removal_tombstone_age_ms(module_id).is_none(),
5814            "a re-added module must not retain a stale removal tombstone"
5815        );
5816    }
5817
5818    /// What one module's owner looked like from the control plane at the
5819    /// instant after its first process was spawned.
5820    #[derive(Debug, PartialEq, Eq)]
5821    struct OwnerInSpawnWindow {
5822        module_id: String,
5823        configured: bool,
5824        on_roster: bool,
5825        admission_refusal: Option<&'static str>,
5826    }
5827
5828    /// A supervised module's process can connect, register, sync its scopes
5829    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5830    /// the module on the roster until afterward; configured supervision does
5831    /// so before spawning. In both cases the owner must
5832    /// already read as configured, so a scoped `route.open` against it is
5833    /// refused as retryable `scope_not_synced` and not as terminal
5834    /// `scope_not_live` ("will never sync").
5835    ///
5836    /// The hook runs just after each initial spawn attempt, before returning to
5837    /// the caller, so no race with a real child's registration is needed.
5838    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5839    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5840        use crate::scopes::ScopeTable;
5841        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5842
5843        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5844        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5845            module_id: module_id.to_string(),
5846            program,
5847            args: Vec::new(),
5848            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5849                .into_iter()
5850                .map(|key| (key.to_string(), dir.path().display().to_string()))
5851                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5852                .collect(),
5853            reserved: false,
5854            reserved_prefixes: Vec::new(),
5855            protocol: ModuleProtocol::Subc,
5856            overlap: Default::default(),
5857        };
5858        let live = super::terminal_history_tests::fake_aft_stub_path();
5859        let missing = dir.path().join("definitely-missing-module");
5860
5861        let handle = SupervisorHandle::new();
5862        let mut supervisor =
5863            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5864                .with_handle(handle.clone());
5865        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5866        let hook_handle = handle.clone();
5867        let hook_observed = Arc::clone(&observed);
5868        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5869            // Exactly what the control plane computes for a scoped route.open
5870            // naming this module as the owner of a scope it has not synced.
5871            let configured = hook_handle.is_configured(module_id);
5872            let selector = ScopeSelector::new(
5873                Principal::Reserved {
5874                    module_id: module_id.to_string(),
5875                },
5876                "s".to_string(),
5877                Some(1),
5878            );
5879            let carrier = Principal::Reserved {
5880                module_id: "carrier".to_string(),
5881            };
5882            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5883                .admit(&carrier, module_id, &selector, configured)
5884            {
5885                Ok(_) => None,
5886                Err(refusal) => Some(refusal.code),
5887            };
5888            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5889                module_id: module_id.to_string(),
5890                configured,
5891                on_roster: hook_handle.get(module_id).is_some(),
5892                admission_refusal,
5893            });
5894        })));
5895
5896        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5897        let configured = supervisor
5898            .supervise_configured(stub("configured", live.clone()), true)
5899            .unwrap();
5900        let with_health = supervisor
5901            .supervise_configured_with_health(
5902                stub("with-health", live.clone()),
5903                true,
5904                HealthConfig::default(),
5905                None,
5906                RestartPolicy::default(),
5907            )
5908            .unwrap();
5909        // The failed-spawn path still puts the module on the roster (as
5910        // failed), so it is configured throughout.
5911        let failed = supervisor
5912            .supervise_configured_with_health(
5913                stub("failed-spawn", missing.clone()),
5914                true,
5915                HealthConfig::default(),
5916                None,
5917                RestartPolicy::default(),
5918            )
5919            .unwrap();
5920        // A failed plain `spawn` puts nothing on the roster, so its mark is
5921        // taken back once the spawn has failed.
5922        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5923
5924        let expected = [
5925            "plain",
5926            "configured",
5927            "with-health",
5928            "failed-spawn",
5929            "spawn-error",
5930        ]
5931        .into_iter()
5932        .map(|module_id| OwnerInSpawnWindow {
5933            module_id: module_id.to_string(),
5934            configured: true,
5935            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5936            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5937        })
5938        .collect::<Vec<_>>();
5939        assert_eq!(*observed.lock().unwrap(), expected);
5940
5941        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5942            assert!(
5943                handle.get(module_id).is_some(),
5944                "{module_id} is on the roster"
5945            );
5946            assert!(
5947                handle.is_configured(module_id),
5948                "{module_id} stays configured"
5949            );
5950        }
5951        assert!(handle.get("spawn-error").is_none());
5952        assert!(
5953            !handle.is_configured("spawn-error"),
5954            "a plain spawn that failed must not leave its module marked configured"
5955        );
5956
5957        // Leaving the roster clears the mark with it.
5958        handle.retire("failed-spawn");
5959        assert!(!handle.is_configured("failed-spawn"));
5960
5961        for module in [plain, configured, with_health] {
5962            module.stop().await.unwrap();
5963        }
5964        drop(failed);
5965    }
5966
5967    /// A configured module reads as starting until the supervisor has admitted
5968    /// its first process (exec confirmed, identity durably recorded), even
5969    /// though that process is already running by then and can register over
5970    /// HELLO, which is what lists it in catalog.list. So a client that needs
5971    /// the supervisor state must wait for that state; seeing the module in
5972    /// the catalog does not mean supervisor.list already says running.
5973    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5974    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5975        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5976        let spec = ModuleSpec {
5977            module_id: "spawned-not-admitted".to_string(),
5978            program: super::terminal_history_tests::fake_aft_stub_path(),
5979            args: Vec::new(),
5980            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5981                .into_iter()
5982                .map(|key| (key.to_string(), dir.path().display().to_string()))
5983                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5984                .collect(),
5985            reserved: false,
5986            reserved_prefixes: Vec::new(),
5987            protocol: ModuleProtocol::Subc,
5988            overlap: Default::default(),
5989        };
5990        let handle = SupervisorHandle::new();
5991        let mut supervisor =
5992            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5993                .with_handle(handle.clone());
5994        let observed = Arc::new(Mutex::new(None));
5995        let hook_handle = handle.clone();
5996        let hook_observed = Arc::clone(&observed);
5997        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5998            let state = hook_handle
5999                .get(module_id)
6000                .expect("a configured module is on the roster before it is spawned")
6001                .state()
6002                .unwrap();
6003            *hook_observed.lock().unwrap() = Some(state);
6004        })));
6005
6006        let module = supervisor.supervise_configured(spec, true).unwrap();
6007        assert_eq!(
6008            *observed.lock().unwrap(),
6009            Some(ModuleState::Starting),
6010            "between the spawn and its admission the supervisor still reports starting"
6011        );
6012        assert_eq!(module.state().unwrap(), ModuleState::Running);
6013        module.stop().await.unwrap();
6014    }
6015
6016    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6017        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6018        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6019            snapshot.process_alive = true;
6020            snapshot.pid = Some(41);
6021            snapshot.spawned_at_ms = Some(42);
6022            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6023            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6024            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6025                device: 43,
6026                inode: 44,
6027                #[cfg(windows)]
6028                windows_image: None,
6029            });
6030        })
6031        .unwrap();
6032        snapshot
6033    }
6034
6035    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6036        let snapshot = lock_snapshot(snapshot).unwrap();
6037        assert!(!snapshot.process_alive);
6038        assert_eq!(snapshot.pid, None);
6039        assert_eq!(snapshot.spawned_at_ms, None);
6040        assert_eq!(snapshot.spawned_from, None);
6041        assert_eq!(snapshot.launch_spec_digest, None);
6042        assert_eq!(snapshot.spawned_file_identity, None);
6043    }
6044
6045    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6046    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6047        let supervisor =
6048            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6049        let mut runtime = supervisor.runtime_config();
6050        runtime.test_seed_stale_facts_before_enable_spawn = true;
6051        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6052        let mut child = None;
6053        let spec = ModuleSpec {
6054            module_id: "failed-enable-clears-facts".to_string(),
6055            program: PathBuf::from("/definitely/missing/failed-enable-module"),
6056            args: Vec::new(),
6057            env: Vec::new(),
6058            reserved: false,
6059            reserved_prefixes: Vec::new(),
6060            protocol: ModuleProtocol::Subc,
6061            overlap: Default::default(),
6062        };
6063
6064        let result = set_child_enabled(
6065            &spec,
6066            &runtime,
6067            &supervisor.registry,
6068            &supervisor.process_liveness,
6069            &snapshot,
6070            &mut child,
6071            true,
6072            &runtime.failure_context(false),
6073        )
6074        .await;
6075
6076        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6077        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6078        assert_snapshot_process_facts_cleared(&snapshot);
6079    }
6080
6081    #[tokio::test]
6082    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6083        let supervisor =
6084            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6085        let runtime = supervisor.runtime_config();
6086        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6087            ModuleState::Restarting,
6088            true,
6089        )));
6090        let spec = ModuleSpec {
6091            module_id: "start-stranded-restarting".to_string(),
6092            program: super::terminal_history_tests::fake_aft_stub_path(),
6093            args: Vec::new(),
6094            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6095            reserved: false,
6096            reserved_prefixes: Vec::new(),
6097            protocol: ModuleProtocol::None,
6098            overlap: Default::default(),
6099        };
6100        let mut child = None;
6101        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6102        assert!(!super::set_child_enabled(
6103            &spec,
6104            &runtime,
6105            &Registry::default(),
6106            &supervisor.process_liveness,
6107            &snapshot,
6108            &mut child,
6109            true,
6110            &runtime.failure_context(false),
6111        )
6112        .await
6113        .unwrap());
6114        assert!(child.is_none());
6115        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6116        assert!(super::set_child_enabled(
6117            &spec,
6118            &runtime,
6119            &Registry::default(),
6120            &supervisor.process_liveness,
6121            &snapshot,
6122            &mut child,
6123            true,
6124            &runtime.failure_context(false),
6125        )
6126        .await
6127        .unwrap());
6128        assert_eq!(
6129            lock_snapshot(&snapshot).unwrap().state,
6130            ModuleState::Running
6131        );
6132        let mut child = child.unwrap();
6133        child.start_kill().unwrap();
6134        child.wait().await.unwrap();
6135    }
6136
6137    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6138    async fn failed_reload_spawn_clears_current_process_facts() {
6139        let supervisor =
6140            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6141        let mut runtime = supervisor.runtime_config();
6142        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6143        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6144        let mut child = None;
6145        let spec = ModuleSpec {
6146            module_id: "failed-reload-clears-facts".to_string(),
6147            program: PathBuf::from("/unused/failed-reload-module"),
6148            args: Vec::new(),
6149            env: Vec::new(),
6150            reserved: false,
6151            reserved_prefixes: Vec::new(),
6152            protocol: ModuleProtocol::Subc,
6153            overlap: Default::default(),
6154        };
6155
6156        let result = handle_reload_spawn_failure(
6157            &spec,
6158            &runtime,
6159            &supervisor.process_liveness,
6160            &snapshot,
6161            &mut child,
6162            "forced reload spawn failure".to_string(),
6163        )
6164        .await;
6165
6166        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6167        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6168        assert_snapshot_process_facts_cleared(&snapshot);
6169    }
6170
6171    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6172    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6173        let supervisor =
6174            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6175        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6176        let module = supervisor.supervised_module(
6177            ModuleSpec {
6178                module_id: "drop-clears-facts".to_string(),
6179                program: PathBuf::from("/unused/drop-module"),
6180                args: Vec::new(),
6181                env: Vec::new(),
6182                reserved: false,
6183                reserved_prefixes: Vec::new(),
6184                protocol: ModuleProtocol::Subc,
6185                overlap: Default::default(),
6186            },
6187            supervisor.runtime_config(),
6188            Arc::clone(&snapshot),
6189            None,
6190        );
6191        assert!(!module
6192            .inner
6193            .monitor
6194            .lock()
6195            .unwrap()
6196            .as_ref()
6197            .unwrap()
6198            .is_finished());
6199
6200        drop(module);
6201
6202        assert_eq!(
6203            lock_snapshot(&snapshot).unwrap().state,
6204            ModuleState::Stopped
6205        );
6206        assert_snapshot_process_facts_cleared(&snapshot);
6207    }
6208
6209    #[cfg(unix)]
6210    #[tokio::test]
6211    async fn rescan_preserves_running_protocol_until_respawn() {
6212        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6213        let initial = ModuleSpec {
6214            module_id: "rescan-protocol".into(),
6215            program: PathBuf::from("/bin/sleep"),
6216            args: vec!["60".into()],
6217            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6218                .into_iter()
6219                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6220                .collect(),
6221            reserved: false,
6222            reserved_prefixes: vec![],
6223            protocol: ModuleProtocol::None,
6224            overlap: Default::default(),
6225        };
6226        let supervisor =
6227            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6228        let module = supervisor.spawn(initial.clone()).unwrap();
6229        assert!(module.status().unwrap().live);
6230        let mut next = initial;
6231        next.protocol = ModuleProtocol::Subc;
6232        module
6233            .update_configuration(next.clone(), HealthConfig::default(), None)
6234            .await
6235            .unwrap();
6236        assert!(
6237            module.status().unwrap().live,
6238            "rescan must not require HELLO from the old non-wire process"
6239        );
6240        let runtime = supervisor.runtime_config();
6241        let action = on_child_exit(
6242            &next,
6243            RestartPolicy::default(),
6244            &supervisor.registry,
6245            &module.inner.snapshot,
6246            &runtime.terminal_ring,
6247            &runtime.spawn_events,
6248            &runtime.child_roster,
6249            ExitReport {
6250                kind: ExitKind::Clean,
6251                code: Some(0),
6252                signal: None,
6253                at_ms: unix_ms_now(),
6254            },
6255            true,
6256        )
6257        .await;
6258        assert!(
6259            matches!(action, NextAction::Restart { .. }),
6260            "the old non-wire process's clean exit must restart"
6261        );
6262        module.drain().await.unwrap();
6263    }
6264
6265    #[cfg(unix)]
6266    #[tokio::test]
6267    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6268        use std::os::unix::fs::PermissionsExt;
6269        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6270        let script = dir.join("module.sh");
6271        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6272        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6273        let record_path = dir.join("live-children.json");
6274        let supervisor =
6275            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6276                .with_live_children_record(&record_path);
6277        for (program, args) in [
6278            (PathBuf::from("sleep"), vec!["60".into()]),
6279            (script, vec![]),
6280        ] {
6281            let spec = ModuleSpec {
6282                module_id: "image-identity".into(),
6283                program: program.clone(),
6284                args,
6285                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6286                    .into_iter()
6287                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6288                    .collect(),
6289                reserved: false,
6290                reserved_prefixes: vec![],
6291                protocol: ModuleProtocol::None,
6292                overlap: Default::default(),
6293            };
6294            let module = supervisor.spawn(spec).unwrap();
6295            #[cfg(target_os = "macos")]
6296            {
6297                // SETEXEC confirmation is asynchronous; the orphan record must
6298                // identify the final image, never the intermediate trampoline.
6299                let deadline = Instant::now() + Duration::from_secs(5);
6300                while crate::live_children::read_record(&record_path)
6301                    .unwrap()
6302                    .iter()
6303                    .all(|entry| entry.executable.is_none())
6304                {
6305                    assert!(Instant::now() < deadline, "module image was not confirmed");
6306                    tokio::time::sleep(Duration::from_millis(5)).await;
6307                }
6308            }
6309            let entry = crate::live_children::read_record(&record_path)
6310                .unwrap()
6311                .pop()
6312                .unwrap();
6313            let observed = subc_os::Process::open(entry.pid)
6314                .unwrap()
6315                .unwrap()
6316                .observe()
6317                .unwrap();
6318            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6319            module.drain().await.unwrap();
6320            assert_eq!(
6321                verdict,
6322                crate::live_children::IdentityVerdict::Matches,
6323                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6324            );
6325        }
6326    }
6327
6328    #[cfg(unix)]
6329    fn http_fixture(
6330        dir: &std::path::Path,
6331        url: &str,
6332        threshold: u32,
6333    ) -> crate::daemon_config::ConfiguredModule {
6334        let path = dir.join("subc.jsonc");
6335        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6336            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6337            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6338            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6339            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6340        }}}).to_string()).unwrap();
6341        crate::daemon_config::load(&path)
6342            .unwrap()
6343            .unwrap()
6344            .modules
6345            .pop()
6346            .unwrap()
6347    }
6348
6349    #[cfg(unix)]
6350    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6351        timeout(Duration::from_secs(5), async {
6352            loop {
6353                if module.status().unwrap().health.status == status {
6354                    break;
6355                }
6356                sleep(Duration::from_millis(5)).await;
6357            }
6358        })
6359        .await
6360        .unwrap_or_else(|_| {
6361            panic!(
6362                "expected {status:?}, got {:?}",
6363                module.status().unwrap().health
6364            )
6365        });
6366    }
6367
6368    #[cfg(unix)]
6369    #[tokio::test]
6370    async fn http_health_status_flips_ok_failing_ok() {
6371        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6372        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6373        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6374        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6375        let serving_status = status.clone();
6376        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6377        let server = tokio::spawn(async move {
6378            loop {
6379                let (mut stream, _) = listener.accept().await.unwrap();
6380                let mut request = [0u8; 2048];
6381                let count = stream.read(&mut request).await.unwrap();
6382                assert!(count > 0, "a probe must send an HTTP request");
6383                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6384                let body = if code == 200 {
6385                    "ready"
6386                } else {
6387                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6388                };
6389                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6390                let _ = stream.write_all(response.as_bytes()).await;
6391            }
6392        });
6393        let configured = http_fixture(&dir, &url, 1000);
6394        let module =
6395            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6396                .supervise_configured_with_health(
6397                    configured.module_spec(),
6398                    true,
6399                    configured.health,
6400                    configured.drain_timeout_ms,
6401                    configured.restart,
6402                )
6403                .unwrap();
6404        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6405        status.store(503, std::sync::atomic::Ordering::SeqCst);
6406        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6407        assert!(module
6408            .status()
6409            .unwrap()
6410            .health
6411            .detail
6412            .unwrap()
6413            .contains("scratch failure"));
6414        status.store(200, std::sync::atomic::Ordering::SeqCst);
6415        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6416        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6417        let before = module.status().unwrap();
6418        let (spec, mut health) = module.configuration().unwrap();
6419        health.http = None;
6420        module
6421            .update_configuration(spec.clone(), health.clone(), Some(10))
6422            .await
6423            .unwrap();
6424        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6425        health.http = Some(url);
6426        module
6427            .update_configuration(spec, health, Some(10))
6428            .await
6429            .unwrap();
6430        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6431        assert_eq!(
6432            module.status().unwrap().pid,
6433            before.pid,
6434            "changing a probe must apply live, not restart its process"
6435        );
6436        let (spec, mut health) = module.configuration().unwrap();
6437        health.failure_threshold = 2;
6438        module
6439            .update_configuration(spec, health, Some(10))
6440            .await
6441            .unwrap();
6442        status.store(503, std::sync::atomic::Ordering::SeqCst);
6443        timeout(Duration::from_secs(5), async {
6444            while module.status().unwrap().spawn_generation == before.spawn_generation {
6445                sleep(Duration::from_millis(5)).await;
6446            }
6447        })
6448        .await
6449        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6450        module.drain().await.unwrap();
6451        server.abort();
6452    }
6453
6454    #[cfg(unix)]
6455    #[tokio::test]
6456    async fn http_health_replacement_recovers_after_one_restart() {
6457        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6458        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6459        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6460        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6461        let mut configured = http_fixture(&dir, &url, 2);
6462        configured.restart.max_restarts = 2;
6463        let module =
6464            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6465                .supervise_configured_with_health(
6466                    configured.module_spec(),
6467                    true,
6468                    configured.health,
6469                    configured.drain_timeout_ms,
6470                    configured.restart,
6471                )
6472                .unwrap();
6473        let first_generation = module.status().unwrap().spawn_generation;
6474        let observed = module.clone();
6475        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6476        let server = tokio::spawn(async move {
6477            let mut generation = first_generation;
6478            let mut probes = 0;
6479            loop {
6480                let (mut stream, _) = listener.accept().await.unwrap();
6481                let mut request = [0; 2048];
6482                assert!(stream.read(&mut request).await.unwrap() > 0);
6483                let current = observed.status().unwrap().spawn_generation;
6484                if current != generation {
6485                    generation = current;
6486                    probes = 0;
6487                }
6488                probes += 1;
6489                // The original process never recovers. Each replacement has one
6490                // transient miss, then answers successfully for its remaining life.
6491                let healthy = generation != first_generation && probes > 1;
6492                let code = if healthy { 200 } else { 503 };
6493                let response = format!(
6494                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6495                );
6496                let _ = stream.write_all(response.as_bytes()).await;
6497                if healthy {
6498                    healthy_tx.send(()).await.unwrap();
6499                }
6500            }
6501        });
6502        timeout(Duration::from_secs(5), async {
6503            let mut healthy_probes = 0;
6504            loop {
6505                if module.state().unwrap() == ModuleState::Failed {
6506                    break;
6507                }
6508                while healthy_rx.try_recv().is_ok() {
6509                    healthy_probes += 1;
6510                }
6511                if healthy_probes >= 3
6512                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6513                {
6514                    break;
6515                }
6516                sleep(Duration::from_millis(5)).await;
6517            }
6518        })
6519        .await
6520        .expect("the replacement must settle healthy without exhausting its restart budget");
6521        let status = module.status().unwrap();
6522        module.drain().await.unwrap();
6523        server.abort();
6524        assert_eq!(status.state, ModuleState::Running);
6525        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6526        assert_eq!(status.health.consecutive_failures, 0);
6527        assert_eq!(status.lifetime_restarts, 1);
6528        assert_eq!(status.restart_count, 1);
6529    }
6530
6531    #[cfg(unix)]
6532    #[tokio::test]
6533    async fn http_health_sustained_failure_exhausts_restart_budget() {
6534        use std::collections::BTreeMap;
6535        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6536        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6537        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6538        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6539        let mut configured = http_fixture(&dir, &url, 2);
6540        configured.restart.max_restarts = 2;
6541        let module =
6542            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6543                .supervise_configured_with_health(
6544                    configured.module_spec(),
6545                    true,
6546                    configured.health,
6547                    configured.drain_timeout_ms,
6548                    configured.restart,
6549                )
6550                .unwrap();
6551        let observed = module.clone();
6552        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6553        let served = probes.clone();
6554        let server = tokio::spawn(async move {
6555            loop {
6556                let (mut stream, _) = listener.accept().await.unwrap();
6557                let mut request = [0; 2048];
6558                assert!(stream.read(&mut request).await.unwrap() > 0);
6559                *served
6560                    .lock()
6561                    .unwrap()
6562                    .entry(observed.status().unwrap().spawn_generation)
6563                    .or_default() += 1;
6564                let _ = stream
6565                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6566                    .await;
6567            }
6568        });
6569        timeout(Duration::from_secs(5), async {
6570            while module.state().unwrap() != ModuleState::Failed {
6571                sleep(Duration::from_millis(5)).await;
6572            }
6573        })
6574        .await
6575        .expect("sustained failures must still exhaust the health restart budget");
6576        let status = module.status().unwrap();
6577        server.abort();
6578        assert_eq!(status.restart_count, 2);
6579        assert_eq!(status.lifetime_restarts, 2);
6580        assert!(!status.process_alive);
6581        assert_eq!(
6582            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6583            vec![2, 2, 2]
6584        );
6585    }
6586
6587    #[cfg(unix)]
6588    #[tokio::test]
6589    async fn spawn_clears_the_previous_process_health() {
6590        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6591        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6592        configured.health.cadence = Duration::from_secs(3600);
6593        let module =
6594            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6595                .supervise_configured_with_health(
6596                    configured.module_spec(),
6597                    false,
6598                    configured.health,
6599                    configured.drain_timeout_ms,
6600                    configured.restart,
6601                )
6602                .unwrap();
6603        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6604            state.health.status = SupervisorHealthStatus::Unresponsive;
6605            state.health.consecutive_failures = 3;
6606            state.health.detail = Some("old process refused connections".into());
6607            state.health.metrics = Some(serde_json::json!({"old": true}));
6608            state.health.last_probe_ms = Some(123);
6609            state.health.last_action = Some("restart".into());
6610            state.health.last_action_ms = Some(124);
6611        })
6612        .unwrap();
6613        module.set_enabled(true).await.unwrap();
6614        let health = module.status().unwrap().health;
6615        module.drain().await.unwrap();
6616        assert_eq!(health.consecutive_failures, 0);
6617        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6618        assert_eq!(health.detail, None);
6619        assert_eq!(health.metrics, None);
6620        assert_eq!(health.last_probe_ms, None);
6621        assert_eq!(health.last_action.as_deref(), Some("restart"));
6622        assert_eq!(health.last_action_ms, Some(124));
6623    }
6624
6625    #[cfg(unix)]
6626    #[tokio::test]
6627    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6628        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6629        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6630        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6631        drop(unused);
6632        let configured = http_fixture(&dir, &url, 2);
6633        let module =
6634            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6635                .supervise_configured_with_health(
6636                    configured.module_spec(),
6637                    true,
6638                    configured.health,
6639                    configured.drain_timeout_ms,
6640                    configured.restart,
6641                )
6642                .unwrap();
6643        let before = module.status().unwrap().spawn_generation;
6644        timeout(Duration::from_secs(5), async {
6645            loop {
6646                let status = module.status().unwrap();
6647                if status.spawn_generation > before {
6648                    assert!(status.lifetime_restarts > 0);
6649                    break;
6650                }
6651                sleep(Duration::from_millis(5)).await;
6652            }
6653        })
6654        .await
6655        .expect("sustained HTTP refusal must trigger the health restart policy");
6656        module.drain().await.unwrap();
6657    }
6658
6659    #[cfg(unix)]
6660    #[tokio::test]
6661    async fn http_health_timeout_honours_deadline() {
6662        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6663        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6664        let server = tokio::spawn(async move {
6665            let _held = listener.accept().await.unwrap();
6666            std::future::pending::<()>().await;
6667        });
6668        let error = timeout(
6669            Duration::from_secs(1),
6670            probe_http_health(&url, Duration::from_millis(10)),
6671        )
6672        .await
6673        .expect("the probe must enforce its own deadline")
6674        .unwrap_err();
6675        server.abort();
6676        assert!(error.to_string().contains("timed out"));
6677    }
6678
6679    #[cfg(unix)]
6680    #[tokio::test]
6681    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6682        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6683        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6684        let url = format!(
6685            "http://localhost:{}/healthz",
6686            listener.local_addr().unwrap().port()
6687        );
6688        let server = tokio::spawn(async move {
6689            let (mut stream, _) = listener.accept().await.unwrap();
6690            let mut request = [0u8; 2048];
6691            assert!(stream.read(&mut request).await.unwrap() > 0);
6692            stream
6693                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6694                .await
6695                .unwrap();
6696        });
6697        // The deadline only bounds a hang. A probe that never tried the IPv6
6698        // address would be refused on 127.0.0.1 and fail at once, so a longer
6699        // deadline does not weaken the assertion; one second timed out under a
6700        // loaded parallel test run.
6701        assert_eq!(
6702            probe_http_health(&url, Duration::from_secs(10))
6703                .await
6704                .unwrap()
6705                .status,
6706            HealthStatus::Ok
6707        );
6708        server.await.unwrap();
6709    }
6710
6711    #[cfg(unix)]
6712    #[tokio::test]
6713    async fn http_health_timeout_keeps_partial_status_and_body() {
6714        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6715        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6716        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6717        let server = tokio::spawn(async move {
6718            let (mut stream, _) = listener.accept().await.unwrap();
6719            let mut request = [0u8; 2048];
6720            assert!(stream.read(&mut request).await.unwrap() > 0);
6721            stream
6722                .write_all(
6723                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6724                )
6725                .await
6726                .unwrap();
6727            std::future::pending::<()>().await;
6728        });
6729        let error = probe_http_health(&url, Duration::from_secs(1))
6730            .await
6731            .unwrap_err()
6732            .to_string();
6733        server.abort();
6734        assert!(
6735            error.contains("timed out")
6736                && error.contains("503 Unavailable")
6737                && error.contains("partial diagnostic"),
6738            "{error}"
6739        );
6740    }
6741
6742    #[cfg(unix)]
6743    #[tokio::test]
6744    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6745        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6746        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6747        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6748        let server = tokio::spawn(async move {
6749            let (mut stream, _) = listener.accept().await.unwrap();
6750            let mut request = [0u8; 2048];
6751            assert!(stream.read(&mut request).await.unwrap() > 0);
6752            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6753            let response = format!(
6754                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6755                body.len()
6756            );
6757            stream.write_all(response.as_bytes()).await.unwrap();
6758        });
6759        let error = probe_http_health(&url, Duration::from_secs(1))
6760            .await
6761            .unwrap_err()
6762            .to_string();
6763        server.await.unwrap();
6764        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6765        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6766        assert!(!error.contains("not-in-diagnostic"));
6767    }
6768
6769    #[cfg(unix)]
6770    #[tokio::test]
6771    async fn http_health_real_nats_server_monitoring() {
6772        if std::process::Command::new("nats-server")
6773            .arg("--version")
6774            .env("XDG_DATA_HOME", std::env::temp_dir())
6775            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6776            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6777            .output()
6778            .is_err()
6779        {
6780            eprintln!(
6781                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6782            );
6783            return;
6784        }
6785        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6786        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6787        let port = monitor.local_addr().unwrap().port();
6788        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6789        let client_port = client.local_addr().unwrap().port();
6790        let config = dir.join("server.conf");
6791        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6792        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6793        configured.program = PathBuf::from("nats-server");
6794        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6795        drop(monitor);
6796        drop(client);
6797        let module =
6798            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6799                .supervise_configured_with_health(
6800                    configured.module_spec(),
6801                    true,
6802                    configured.health,
6803                    configured.drain_timeout_ms,
6804                    configured.restart,
6805                )
6806                .unwrap();
6807        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6808        module.drain().await.unwrap();
6809    }
6810
6811    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6812    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6813        let supervisor =
6814            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6815        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6816        let initial = ModuleSpec {
6817            module_id: "rescan-preserves-spawn-facts".to_string(),
6818            program: PathBuf::from("/spawned/module"),
6819            args: Vec::new(),
6820            env: Vec::new(),
6821            reserved: false,
6822            reserved_prefixes: Vec::new(),
6823            protocol: ModuleProtocol::Subc,
6824            overlap: Default::default(),
6825        };
6826        let module = supervisor.supervised_module(
6827            initial.clone(),
6828            supervisor.runtime_config(),
6829            snapshot,
6830            None,
6831        );
6832        let before = module.status().unwrap();
6833        let mut replacement = initial;
6834        replacement.program = PathBuf::from("/rescanned/replacement-module");
6835
6836        module
6837            .update_configuration(replacement, HealthConfig::default(), None)
6838            .await
6839            .unwrap();
6840
6841        let after = module.status().unwrap();
6842        assert_eq!(after.pid, before.pid);
6843        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6844        assert_eq!(after.spawned_from, before.spawned_from);
6845        drop(module);
6846    }
6847}
6848
6849fn unix_ms_now() -> u64 {
6850    SystemTime::now()
6851        .duration_since(UNIX_EPOCH)
6852        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6853        .unwrap_or(0)
6854}
6855
6856async fn supervise_loop(
6857    mut spec: ModuleSpec,
6858    mut runtime: SupervisorRuntimeConfig,
6859    registry: Arc<Registry>,
6860    process_liveness: Arc<SupervisorProcessLiveness>,
6861    snapshot: SharedSnapshot,
6862    mut child: Option<SupervisedChild>,
6863    mut commands: mpsc::Receiver<SupervisorCommand>,
6864) {
6865    let mut health_probe = HealthProbeRuntime::default();
6866    // Registry writes (including embedded callers) notify this module only.
6867    // Subscribe before the first refresh; watch retains changes that arrive
6868    // while commands or probes are running. No polling fallback is needed.
6869    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6870        Ok(changes) => Some(changes),
6871        Err(err) => {
6872            // A poisoned registry cannot accept further writes, so it cannot
6873            // recover via a timer. Keep exit and command handling alive.
6874            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6875            None
6876        }
6877    };
6878    // All restart backoffs run here, including health and operator requests.
6879    // While one is pending the loop serves commands, so disable or drain can
6880    // cancel the replacement without spawning a process just to stop it.
6881    let mut pending_respawn: Option<PendingRespawn> = None;
6882    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6883    // before anything else so a stop that interrupted a swap runs at once.
6884    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6885    let mut previous_instance_recheck = tokio::time::interval_at(
6886        Instant::now() + Duration::from_secs(5),
6887        Duration::from_secs(5),
6888    );
6889    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6890    loop {
6891        #[cfg(test)]
6892        {
6893            lock_snapshot(&snapshot).unwrap().actor_select = None;
6894        }
6895        #[cfg(target_os = "macos")]
6896        if let Some(active) = child.as_mut() {
6897            active.confirm_privacy_exec().await;
6898        }
6899        if let Some(scheduled) = runtime
6900            .scheduled_respawn
6901            .lock()
6902            .unwrap_or_else(|p| p.into_inner())
6903            .take()
6904        {
6905            runtime.recovery_is_daemon = scheduled.daemon_initiated;
6906            pending_respawn = Some(scheduled);
6907        }
6908        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6909            pending_respawn = None;
6910            cancel_deferred_reload(
6911                &runtime,
6912                &spec.module_id,
6913                "respawn cancelled by a supervisor command",
6914            );
6915        }
6916        if child.is_none()
6917            && pending_respawn.is_none()
6918            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6919        {
6920            cancel_deferred_reload(
6921                &runtime,
6922                &spec.module_id,
6923                "respawn cancelled before a replacement was spawned",
6924            );
6925            let mut stranded = false;
6926            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6927                state.respawn_pending = false;
6928                state.coalesced_restart_pending = false;
6929                if matches!(
6930                    state.state,
6931                    ModuleState::Restarting
6932                        | ModuleState::Starting
6933                        | ModuleState::Draining
6934                        | ModuleState::Unresponsive
6935                ) {
6936                    error!(module_id = %spec.module_id, state = ?state.state,
6937                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6938                    stranded = true;
6939                }
6940            });
6941            if stranded {
6942                fail_snapshot(
6943                    &snapshot,
6944                    Some(&spec.module_id),
6945                    None,
6946                    &runtime.failure_context(runtime.recovery_is_daemon),
6947                );
6948            }
6949        }
6950        if let Some(command) = requeued.pop_front() {
6951            if !handle_supervisor_command(
6952                command,
6953                &mut spec,
6954                &mut runtime,
6955                &registry,
6956                &process_liveness,
6957                &snapshot,
6958                &mut child,
6959                &mut commands,
6960                &mut requeued,
6961            )
6962            .await
6963            {
6964                return;
6965            }
6966            if child.is_some() || !respawn_still_pending(&snapshot) {
6967                pending_respawn = None;
6968                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6969                    state.respawn_pending = false
6970                });
6971            }
6972            continue;
6973        }
6974        if child.is_some() {
6975            runtime.recovery_is_daemon = true;
6976            #[cfg(test)]
6977            {
6978                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6979            }
6980            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6981            let active = child.as_mut().expect("child checked above");
6982            if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6983                active.registration_deadline = None;
6984            }
6985            let registration_deadline = active.registration_deadline;
6986            let no_checks = runtime.health.http.is_none()
6987                && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6988                    || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6989            let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6990                state
6991                    .failed_retry
6992                    .uptime_reset_at(no_checks, runtime.restart_policy.window)
6993            });
6994            let wake_at = match (health_probe.wake_at(), uptime_reset) {
6995                (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6996                (probe, None) => probe,
6997            };
6998            let wake_at = match (wake_at, registration_deadline) {
6999                (Some(wake), Some(register)) => Some(wake.min(register)),
7000                (wake, register) => wake.or(register),
7001            };
7002            #[cfg(test)]
7003            let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7004            let probe_sleep = supervisor_sleep(wake_at);
7005            tokio::pin!(probe_sleep);
7006            // A `protocol: "none"` child (a plain process that never registers
7007            // over the subc wire, such as nats-server) with no HTTP health
7008            // check has nothing to probe, so it waits only for its exit or a
7009            // supervisor command. A rescan that adds an HTTP check is a command.
7010            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7011            #[cfg(test)]
7012            {
7013                let registration_pending = wire_child
7014                    && registration_changes
7015                        .as_ref()
7016                        .is_some_and(|changes| changes.has_changed().unwrap());
7017                let mut state = lock_snapshot(&snapshot).unwrap();
7018                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7019                    generation: state.spawn_generation,
7020                    turn: state.actor_turns,
7021                    registered_connection: health_probe.registered_connection,
7022                    next_probe_at: health_probe.next_probe_at,
7023                    wake_after,
7024                });
7025            }
7026            let active_child = child.as_mut().expect("child checked above");
7027            tokio::select! {
7028                wait_result = active_child.wait() => {
7029                    if let Ok(mut state) = lock_snapshot(&snapshot) {
7030                        state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7031                    }
7032                    #[cfg(test)]
7033                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7034                    // Every arm below that gives up on the CHILD must keep the
7035                    // supervision task itself alive (child = None, loop
7036                    // continues into command-serving mode). Returning here
7037                    // closes the command channel, which makes the module
7038                    // permanently unrestartable in-band: a clean child exit
7039                    // of an enabled module once wedged the fleet this way
7040                    // ('supervisor command channel is closed') and required a
7041                    // full daemon restart to recover.
7042                    let exit_report = match wait_result {
7043                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7044                        Err(err) => {
7045                            active_child.drain_stderr(&spec.module_id).await;
7046                            fail_after_wait_error(
7047                                &spec.module_id,
7048                                &snapshot,
7049                                &runtime.terminal_ring,
7050                                &runtime.spawn_events,
7051                                format!("failed to wait for supervised module: {err}"),
7052                            &runtime.failure_context(true),
7053                            );
7054                            untrack_if_registration_released(
7055                                &process_liveness,
7056                                &registry,
7057                                &spec.module_id,
7058                                &snapshot,
7059                            );
7060                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7061                            child = None;
7062                            continue;
7063                        }
7064                    };
7065                    active_child.drain_stderr(&spec.module_id).await;
7066
7067                    let next = on_child_exit(
7068                        &spec,
7069                        runtime.restart_policy,
7070                        &registry,
7071                        &snapshot,
7072                        &runtime.terminal_ring,
7073                        &runtime.spawn_events,
7074                        &runtime.child_roster,
7075                        exit_report,
7076                        true,
7077                    ).await;
7078                    runtime.start_requester = None;
7079                    runtime.pending_activity_restart = false;
7080                    // The exit is recorded, so a daemon shutdown may stop
7081                    // waiting for this child (see `SupervisedChild::wait`).
7082                    active_child.release_roster();
7083                    match next {
7084                        NextAction::Stop { registration_released } => {
7085                            if registration_released {
7086                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7087                            }
7088                            child = None;
7089                        }
7090                        NextAction::Restart { schedule } => {
7091                            let delay = schedule.map_or(
7092                                runtime.restart_policy.delay_for_restart(0),
7093                                |schedule| schedule.delay,
7094                            );
7095                            if let Some(schedule) = schedule {
7096                                log_crash_respawn(&spec.module_id, schedule);
7097                            }
7098                            // The exited child is fully recorded at this point,
7099                            // so release it and count the backoff down in the
7100                            // command-serving branch below rather than sleeping
7101                            // here: commands cannot be received from inside this
7102                            // select arm, and an operator disable or drain that
7103                            // arrives during the backoff must cancel the pending
7104                            // respawn instead of waiting for it to spawn first.
7105                            child = None;
7106                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7107                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7108                        }
7109                    }
7110                }
7111                command = commands.recv() => {
7112                    #[cfg(test)]
7113                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7114                    let Some(command) = command else {
7115                        return;
7116                    };
7117                    if !handle_supervisor_command(
7118                        command,
7119                        &mut spec,
7120                        &mut runtime,
7121                        &registry,
7122                        &process_liveness,
7123                        &snapshot,
7124                        &mut child,
7125                        &mut commands,
7126                        &mut requeued,
7127                    ).await {
7128                        return;
7129                    }
7130                }
7131                _ = async {
7132                    match registration_changes.as_mut() {
7133                        Some(changes) => { let _ = changes.changed().await; }
7134                        None => std::future::pending().await,
7135                    }
7136                }, if wire_child => {
7137                    #[cfg(test)]
7138                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7139                }
7140                _ = &mut probe_sleep => {
7141                    #[cfg(test)]
7142                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7143                    if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7144                        // Recheck after the wake: HELLO, exit or shutdown may
7145                        // have become ready in the same executor tick.
7146                        active_child.registration_deadline = None;
7147                        if !runtime.child_roster.is_closed()
7148                            && !active_child.registered_since_spawn()
7149                            && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7150                        {
7151                            warn!(module_id = %spec.module_id, pid = active_child.pid,
7152                                wait_secs = runtime.register_timeout.as_secs(),
7153                                "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7154                            active_child.spawn_failure = Some("registration_timeout".into());
7155                            if let Err(error) = stop_unregistered_child(active_child).await {
7156                                error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7157                                fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7158                                    &runtime.spawn_events, format!("registration_timeout: {error}"),
7159                                    &runtime.failure_context(true));
7160                                child = None;
7161                            }
7162                            // The ordinary exit handler records the child's exit,
7163                            // spends crash budget, and schedules recovery.
7164                            continue;
7165                        }
7166                    }
7167                    if health_probe.due() {
7168                        run_health_probe_cycle(
7169                            &spec,
7170                            &runtime,
7171                            &registry,
7172                            &process_liveness,
7173                            &snapshot,
7174                            &mut child,
7175                        ).await;
7176                        if child.is_none() {
7177                            runtime.start_requester = None;
7178                            runtime.pending_activity_restart = false;
7179                        }
7180                        if child.is_some() {
7181                            health_probe.schedule_next(&spec, runtime.health.cadence);
7182                        }
7183                    }
7184                }
7185            }
7186        } else if let Some(pending) = pending_respawn {
7187            tokio::select! {
7188                _ = sleep_until(pending.deadline) => {
7189                    pending_respawn = None;
7190                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7191                    // A command handled below while the backoff elapsed may
7192                    // have stopped the module; never respawn past an operator's
7193                    // disable or drain.
7194                    if !respawn_still_pending(&snapshot) {
7195                        continue;
7196                    }
7197                    // The daemon began shutting down during the backoff: the
7198                    // spawn would be refused anyway, and refusing it here
7199                    // leaves the module stopped instead of reporting a
7200                    // failed restart.
7201                    if runtime.child_roster.is_closed() {
7202                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7203                            state.state = ModuleState::Stopped;
7204                        });
7205                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7206                        continue;
7207                    }
7208                    if let Err(err) = release_dead_registration(
7209                        &registry,
7210                        runtime.forwarding.as_deref(),
7211                        &snapshot,
7212                        &spec.module_id,
7213                        &runtime.failure_context(runtime.recovery_is_daemon),
7214                    ).await {
7215                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7216                        continue;
7217                    }
7218
7219                    if matches!(pending.kind, RespawnKind::Reload) {
7220                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7221                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
7222                        if result.is_ok() && runtime.pending_activity_restart {
7223                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7224                        }
7225                        runtime.pending_activity_restart = false;
7226                        if let Some(reply) = reply { let _ = reply.send(result); }
7227                        continue;
7228                    }
7229                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7230                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7231                        Ok(next_child) => {
7232                            child = Some(next_child);
7233                            if runtime.pending_activity_restart {
7234                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7235                                runtime.pending_activity_restart = false;
7236                            }
7237                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7238                        }
7239                        Err(err) => {
7240                            fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7241                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7242                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7243                        }
7244                    }
7245                }
7246                command = commands.recv() => {
7247                    let Some(command) = command else {
7248                        return;
7249                    };
7250                    if !handle_supervisor_command(
7251                        command,
7252                        &mut spec,
7253                        &mut runtime,
7254                        &registry,
7255                        &process_liveness,
7256                        &snapshot,
7257                        &mut child,
7258                        &mut commands,
7259                        &mut requeued,
7260                    ).await {
7261                        return;
7262                    }
7263                    // Reconcile the pending respawn with what the command did:
7264                    // a start may already have spawned a fresh child,
7265                    // while a disable or drain moved the snapshot out of the
7266                    // state the respawn was counting down from.
7267                    if child.is_some() || !respawn_still_pending(&snapshot) {
7268                        pending_respawn = None;
7269                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7270                    }
7271                }
7272            }
7273        } else {
7274            let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7275                (state.enabled && state.state == ModuleState::Failed)
7276                    .then_some(state.failed_retry.pending)
7277                    .flatten()
7278            });
7279            let command = if lock_snapshot(&snapshot)
7280                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7281            {
7282                tokio::select! {
7283                    command = commands.recv() => command,
7284                    _ = previous_instance_recheck.tick() => {
7285                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7286                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7287                            if enabled && !runtime.child_roster.is_closed() {
7288                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7289                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7290                                    Ok(next_child) => child = Some(next_child),
7291                                    Err(error) => {
7292                                        fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7293                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7294                                    }
7295                                }
7296                            } else {
7297                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7298                                    state.waiting_for_previous_pid = None;
7299                                    state.waiting_for_previous_detail = None;
7300                                });
7301                            }
7302                        }
7303                        continue;
7304                    }
7305                }
7306            } else {
7307                tokio::select! {
7308                    biased;
7309                    command = commands.recv() => command,
7310                    _ = async {
7311                        match slow_retry {
7312                            Some(retry) => sleep_until(retry.deadline).await,
7313                            None => std::future::pending().await,
7314                        }
7315                    } => {
7316                        let retry = slow_retry.expect("only a scheduled slow retry can wake");
7317                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7318                            state.failed_retry.pending = None;
7319                        });
7320                        if runtime.child_roster.is_closed() {
7321                            continue;
7322                        }
7323                        info!(module_id = %spec.module_id, attempt = retry.attempt,
7324                            delay_secs = retry.delay.as_secs(), "starting failed module retry");
7325                        if let Err(error) = set_child_enabled(
7326                            &spec, &runtime, &registry, &process_liveness, &snapshot, &mut child, true,
7327                            &runtime.failure_context(true),
7328                        ).await {
7329                            error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7330                        }
7331                        continue;
7332                    }
7333                }
7334            };
7335            let Some(command) = command else {
7336                return;
7337            };
7338            if !handle_supervisor_command(
7339                command,
7340                &mut spec,
7341                &mut runtime,
7342                &registry,
7343                &process_liveness,
7344                &snapshot,
7345                &mut child,
7346                &mut commands,
7347                &mut requeued,
7348            )
7349            .await
7350            {
7351                return;
7352            }
7353        }
7354    }
7355}
7356
7357// Preserve the deadline until the future is polled. Starting a relative sleep
7358// after the executor yields would restart part of the countdown.
7359async fn supervisor_sleep(wake_at: Option<Instant>) {
7360    match wake_at {
7361        Some(deadline) => sleep_until(deadline).await,
7362        None => std::future::pending().await,
7363    }
7364}
7365
7366fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7367    info!(
7368        module_id,
7369        restart_in_window = schedule.restart_in_window,
7370        delay_ms = schedule.delay.as_millis() as u64,
7371        "respawning after crash"
7372    );
7373}
7374
7375/// Whether the respawn a backoff was counting down to is still wanted. A
7376/// disable or drain handled while the backoff elapsed moves the snapshot out
7377/// of `Restarting`, and the operator's stop must win over the pending respawn,
7378/// so every sleep-then-spawn path re-validates against the live snapshot
7379/// instead of assuming the state it left behind still holds.
7380fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7381    matches!(
7382        lock_snapshot(snapshot),
7383        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7384    )
7385}
7386
7387enum NextAction {
7388    Stop {
7389        registration_released: bool,
7390    },
7391    Restart {
7392        schedule: Option<CrashRestartSchedule>,
7393    },
7394}
7395
7396#[allow(clippy::too_many_arguments)]
7397async fn handle_supervisor_command(
7398    command: SupervisorCommand,
7399    spec: &mut ModuleSpec,
7400    runtime: &mut SupervisorRuntimeConfig,
7401    registry: &Arc<Registry>,
7402    process_liveness: &SupervisorProcessLiveness,
7403    snapshot: &SharedSnapshot,
7404    child: &mut Option<SupervisedChild>,
7405    commands: &mut mpsc::Receiver<SupervisorCommand>,
7406    requeued: &mut VecDeque<SupervisorCommand>,
7407) -> bool {
7408    if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7409        runtime.recovery_is_daemon = false;
7410    }
7411    if matches!(
7412        command,
7413        SupervisorCommand::Drain { .. }
7414            | SupervisorCommand::Retire { .. }
7415            | SupervisorCommand::SetEnabled { .. }
7416            | SupervisorCommand::Restart { .. }
7417    ) {
7418        let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7419            state.failed_retry.cancel()
7420        });
7421    }
7422    let command =
7423        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7424            match command {
7425                SupervisorCommand::Restart { reply, .. }
7426                | SupervisorCommand::Reload { reply, .. }
7427                | SupervisorCommand::Swap { reply, .. } => {
7428                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7429                    return true;
7430                }
7431                SupervisorCommand::SetEnabled {
7432                    enabled: true,
7433                    reply,
7434                    ..
7435                } => {
7436                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7437                    return true;
7438                }
7439                other => other,
7440            }
7441        } else {
7442            command
7443        };
7444    match command {
7445        SupervisorCommand::Drain { requester, reply } => {
7446            // A plain stop runs no forwarding drain, so nothing reaches the
7447            // module over its connection before the wait: ask by signal.
7448            let result = drain_optional_child(
7449                &spec.module_id,
7450                spec.protocol,
7451                StopNotice::NotSent,
7452                registry,
7453                runtime.forwarding.as_deref(),
7454                snapshot,
7455                &runtime.terminal_ring,
7456                &runtime.spawn_events,
7457                child,
7458                runtime.drain_timeout,
7459                ModuleState::Stopped,
7460                None,
7461                &FailureContext::operator(),
7462            )
7463            .await;
7464            let registration_released = result.is_ok();
7465            if registration_released {
7466                runtime.record_activity_transition(
7467                    "subc.module_stopped",
7468                    &spec.module_id,
7469                    requester,
7470                );
7471            }
7472            let _ = reply.send(result);
7473            if registration_released {
7474                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7475            }
7476            false
7477        }
7478        SupervisorCommand::Retire { requester, reply } => {
7479            let result = async {
7480                let stop_notice = begin_forwarding_drain_if_configured(
7481                    spec,
7482                    runtime,
7483                    registry,
7484                    snapshot,
7485                    None,
7486                    RouteCloseReason::Disable,
7487                )
7488                .await?;
7489                drain_optional_child(
7490                    &spec.module_id,
7491                    spec.protocol,
7492                    stop_notice,
7493                    registry,
7494                    runtime.forwarding.as_deref(),
7495                    snapshot,
7496                    &runtime.terminal_ring,
7497                    &runtime.spawn_events,
7498                    child,
7499                    runtime.drain_timeout,
7500                    ModuleState::Stopped,
7501                    None,
7502                    &FailureContext::operator(),
7503                )
7504                .await
7505            }
7506            .await;
7507            let registration_released = result.is_ok();
7508            if registration_released {
7509                runtime.record_activity_transition(
7510                    "subc.module_stopped",
7511                    &spec.module_id,
7512                    requester,
7513                );
7514            }
7515            let _ = reply.send(result);
7516            if registration_released {
7517                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7518            }
7519            false
7520        }
7521        SupervisorCommand::Restart {
7522            requester,
7523            drain_timeout_ms,
7524            received_at_generation,
7525            queued_at,
7526            reply,
7527        } => {
7528            // Without this line a restart that waited in the queue (behind a
7529            // health probe cycle or another command) was invisible: the log
7530            // showed only the drain timing out, minutes after the operator's call.
7531            info!(
7532                module_id = %spec.module_id,
7533                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7534                "restart command dequeued"
7535            );
7536            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7537            // caller whose own request lane rides the module being restarted: the
7538            // caller's in-flight request keeps the drain from quiescing, the drain
7539            // keeps the restart from completing, and the completion keeps the reply
7540            // from releasing the caller — so the drain always timed out and cut the
7541            // initiator with a GOODBYE, even on a healthy module. Replying once the
7542            // restart is validated lets a self-lane caller settle, which is exactly
7543            // what makes the drain succeed. Completion is observable via
7544            // supervisor.list / module status; a post-ack failure lands the module
7545            // in a visible terminal state below rather than in a reply nobody can
7546            // receive.
7547            let validation = match lock_snapshot(snapshot) {
7548                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7549                    module_id: spec.module_id.clone(),
7550                }),
7551                Ok(_) => Ok(()),
7552                Err(err) => Err(err),
7553            };
7554            let initiated = validation.is_ok();
7555            let _ = reply.send(validation);
7556            // A restart asks for a fresh process. Commands run one at a time,
7557            // so a restart queued behind another restart (two operator calls
7558            // in quick succession) is dequeued the moment the first one has
7559            // spawned its replacement -- before that process has sent HELLO.
7560            // Running it would drain and kill the process the first restart
7561            // just produced, which is the opposite of what both callers asked
7562            // for. If a process spawned after this request was received is
7563            // still supervised, the request is already satisfied. Not when the
7564            // configuration changed since that spawn: then the newer process
7565            // predates the spec this restart may exist to apply.
7566            // Generation publication can lag a child that is already serving
7567            // while its identity is recorded. Only process creation after request
7568            // receipt satisfies a restart, not publication after receipt.
7569            let satisfied_by_generation = if initiated
7570                && child
7571                    .as_ref()
7572                    .is_some_and(|child| child.spawned_at > queued_at)
7573            {
7574                lock_snapshot(snapshot).ok().and_then(|state| {
7575                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7576                })
7577            } else {
7578                None
7579            };
7580            let satisfied_by_pending = initiated
7581                && child.is_none()
7582                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7583                    // No child exists yet, so this replacement will necessarily
7584                    // be created after the queued request, unlike a published
7585                    // generation whose process may already have been serving.
7586                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7587                    if pending {
7588                        state.coalesced_restart_pending = true;
7589                    }
7590                    pending
7591                });
7592            if satisfied_by_pending {
7593                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7594            } else if let Some(generation) = satisfied_by_generation {
7595                info!(
7596                    module_id = %spec.module_id,
7597                    received_at_generation,
7598                    "restart already satisfied by generation {generation}; not restarting again"
7599                );
7600            } else if initiated {
7601                // Precedence: this restart's operator override, else the module's
7602                // configured budget (already resolved into the runtime).
7603                let drain_timeout = drain_timeout_ms
7604                    .map(Duration::from_millis)
7605                    .unwrap_or(runtime.drain_timeout);
7606                if let Err(err) = restart_child(
7607                    spec,
7608                    runtime,
7609                    registry,
7610                    process_liveness,
7611                    snapshot,
7612                    child,
7613                    drain_timeout,
7614                )
7615                .await
7616                {
7617                    warn!(
7618                        module_id = %spec.module_id,
7619                        error = %err,
7620                        "operator restart failed after initiation ack; module state carries the outcome"
7621                    );
7622                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7623                        state.state = ModuleState::Failed;
7624                        clear_current_process_facts(state);
7625                    });
7626                } else {
7627                    runtime.start_requester = requester;
7628                    runtime.pending_activity_restart = true;
7629                }
7630            }
7631            true
7632        }
7633        SupervisorCommand::Reload { requester, reply } => {
7634            let result =
7635                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7636            if result.is_ok() {
7637                runtime.start_requester = requester;
7638                runtime.pending_activity_restart = true;
7639            }
7640            if result.is_ok()
7641                && runtime
7642                    .scheduled_respawn
7643                    .lock()
7644                    .unwrap_or_else(|p| p.into_inner())
7645                    .as_ref()
7646                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7647            {
7648                *runtime
7649                    .deferred_reload_reply
7650                    .lock()
7651                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7652            } else {
7653                let _ = reply.send(result);
7654            }
7655            true
7656        }
7657        SupervisorCommand::SetEnabled {
7658            enabled,
7659            requester,
7660            reply,
7661        } => {
7662            let previous_requester = runtime.start_requester.clone();
7663            if enabled {
7664                runtime.start_requester = requester.clone();
7665            }
7666            let result = set_child_enabled(
7667                spec,
7668                runtime,
7669                registry,
7670                process_liveness,
7671                snapshot,
7672                child,
7673                enabled,
7674                &runtime.failure_context(false),
7675            )
7676            .await;
7677            if matches!(result, Ok(true)) {
7678                runtime.pending_activity_restart = false;
7679                if enabled {
7680                    runtime.start_requester = requester;
7681                } else {
7682                    runtime.start_requester = None;
7683                    runtime.record_activity_transition(
7684                        "subc.module_disabled",
7685                        &spec.module_id,
7686                        requester,
7687                    );
7688                }
7689            } else {
7690                runtime.start_requester = previous_requester;
7691            }
7692            let _ = reply.send(result);
7693            true
7694        }
7695        SupervisorCommand::UpdateConfiguration {
7696            spec: next_spec,
7697            health,
7698            drain_timeout_ms,
7699            reply,
7700        } => {
7701            if let Some(handle) = &runtime.supervisor_handle {
7702                handle.apply_identity_configuration(&next_spec);
7703            }
7704            *spec = next_spec;
7705            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7706                state.configuration_updated_since_spawn = true;
7707            });
7708            let health_changed = runtime.health != health;
7709            runtime.health = health;
7710            // Reset the cadence and old endpoint's failure streak on a live
7711            // health-policy change rather than waiting for its old deadline.
7712            if health_changed {
7713                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7714                    state.health = ModuleHealthStatus::default();
7715                });
7716            }
7717            runtime.drain_timeout = drain_timeout_ms
7718                .map(Duration::from_millis)
7719                .unwrap_or(runtime.default_drain_timeout);
7720            *runtime
7721                .effective_drain_timeout
7722                .lock()
7723                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7724            let _ = reply.send(());
7725            true
7726        }
7727        SupervisorCommand::Swap {
7728            requester,
7729            ready_timeout,
7730            reply,
7731        } => {
7732            let mut swap_runtime = runtime.clone();
7733            swap_runtime.start_requester = requester;
7734            let before = lock_snapshot(snapshot)
7735                .ok()
7736                .map(|state| state.spawn_generation);
7737            let end = swap::run_swap(
7738                spec,
7739                &swap_runtime,
7740                registry,
7741                process_liveness,
7742                snapshot,
7743                child,
7744                commands,
7745                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7746                reply,
7747            )
7748            .await;
7749            if before
7750                != lock_snapshot(snapshot)
7751                    .ok()
7752                    .map(|state| state.spawn_generation)
7753            {
7754                runtime.start_requester = swap_runtime.start_requester;
7755            }
7756            requeued.extend(end.requeue);
7757            true
7758        }
7759    }
7760}
7761
7762async fn restart_child(
7763    spec: &ModuleSpec,
7764    runtime: &SupervisorRuntimeConfig,
7765    registry: &Registry,
7766    process_liveness: &SupervisorProcessLiveness,
7767    snapshot: &SharedSnapshot,
7768    child: &mut Option<SupervisedChild>,
7769    drain_timeout: Duration,
7770) -> Result<(), SuperviseError> {
7771    // Restart cycles a running module; it must not silently start a disabled one.
7772    if !lock_snapshot(snapshot)?.enabled {
7773        return Err(SuperviseError::Disabled {
7774            module_id: spec.module_id.clone(),
7775        });
7776    }
7777    let stop_notice = begin_forwarding_drain_with_timeout(
7778        spec,
7779        runtime,
7780        registry,
7781        snapshot,
7782        None,
7783        RouteCloseReason::Restart,
7784        drain_timeout,
7785    )
7786    .await?;
7787
7788    if child.is_some() {
7789        drain_optional_child(
7790            &spec.module_id,
7791            spec.protocol,
7792            stop_notice,
7793            registry,
7794            runtime.forwarding.as_deref(),
7795            snapshot,
7796            &runtime.terminal_ring,
7797            &runtime.spawn_events,
7798            child,
7799            drain_timeout,
7800            ModuleState::Restarting,
7801            Some(true),
7802            &FailureContext::operator(),
7803        )
7804        .await?;
7805    } else {
7806        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7807            state.enabled = true;
7808            state.state = ModuleState::Restarting;
7809            clear_current_process_facts(state);
7810        })?;
7811        release_dead_registration(
7812            registry,
7813            runtime.forwarding.as_deref(),
7814            snapshot,
7815            &spec.module_id,
7816            &runtime.failure_context(false),
7817        )
7818        .await?;
7819    }
7820
7821    reset_restart_count(snapshot, &spec.module_id)?;
7822    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7823    schedule_respawn(
7824        runtime,
7825        snapshot,
7826        &spec.module_id,
7827        runtime.restart_policy.backoff,
7828        RespawnKind::Spawn,
7829        false,
7830    )
7831}
7832
7833async fn reload_child(
7834    spec: &ModuleSpec,
7835    runtime: &SupervisorRuntimeConfig,
7836    registry: &Registry,
7837    process_liveness: &SupervisorProcessLiveness,
7838    snapshot: &SharedSnapshot,
7839    child: &mut Option<SupervisedChild>,
7840) -> Result<(), SuperviseError> {
7841    // Reload cycles a running module; it must not silently start a disabled one.
7842    if !lock_snapshot(snapshot)?.enabled {
7843        return Err(SuperviseError::Disabled {
7844            module_id: spec.module_id.clone(),
7845        });
7846    }
7847    let stop_notice = begin_forwarding_drain(
7848        spec,
7849        runtime,
7850        registry,
7851        snapshot,
7852        Some(true),
7853        RouteCloseReason::Reload,
7854    )
7855    .await?;
7856
7857    if child.is_some() {
7858        drain_optional_child(
7859            &spec.module_id,
7860            spec.protocol,
7861            stop_notice,
7862            registry,
7863            runtime.forwarding.as_deref(),
7864            snapshot,
7865            &runtime.terminal_ring,
7866            &runtime.spawn_events,
7867            child,
7868            runtime.drain_timeout,
7869            ModuleState::Restarting,
7870            Some(true),
7871            &FailureContext::operator(),
7872        )
7873        .await?;
7874    } else {
7875        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7876            state.enabled = true;
7877            state.state = ModuleState::Restarting;
7878            clear_current_process_facts(state);
7879        })?;
7880        release_dead_registration(
7881            registry,
7882            runtime.forwarding.as_deref(),
7883            snapshot,
7884            &spec.module_id,
7885            &runtime.failure_context(false),
7886        )
7887        .await?;
7888    }
7889
7890    reset_restart_count(snapshot, &spec.module_id)?;
7891    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7892    schedule_respawn(
7893        runtime,
7894        snapshot,
7895        &spec.module_id,
7896        runtime.restart_policy.backoff,
7897        RespawnKind::Reload,
7898        false,
7899    )
7900}
7901
7902async fn finish_reload_child(
7903    spec: &ModuleSpec,
7904    runtime: &SupervisorRuntimeConfig,
7905    registry: &Registry,
7906    process_liveness: &SupervisorProcessLiveness,
7907    snapshot: &SharedSnapshot,
7908    child: &mut Option<SupervisedChild>,
7909) -> Result<(), SuperviseError> {
7910    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7911    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7912        Ok(next_child) => next_child,
7913        Err(err) => {
7914            return handle_reload_spawn_failure(
7915                spec,
7916                runtime,
7917                process_liveness,
7918                snapshot,
7919                child,
7920                format!("new child failed to spawn: {err}"),
7921            )
7922            .await;
7923        }
7924    };
7925    *child = Some(next_child);
7926
7927    let wait_outcome = {
7928        let active_child = child.as_mut().expect("new reload child was just stored");
7929        wait_for_registration_after_reload(
7930            registry,
7931            &spec.module_id,
7932            snapshot,
7933            active_child,
7934            REGISTRY_RELEASE_TIMEOUT,
7935        )
7936        .await?
7937    };
7938
7939    match wait_outcome {
7940        RegistrationWaitOutcome::Registered => {
7941            child
7942                .as_mut()
7943                .expect("registered reload child")
7944                .registration_deadline = None;
7945            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7946            Ok(())
7947        }
7948        RegistrationWaitOutcome::Exited(exit_report) => {
7949            if let Some(active_child) = child.as_mut() {
7950                active_child.drain_stderr(&spec.module_id).await;
7951            }
7952            // Keep the reaped child's roster guard until its terminal is written.
7953            // Shutdown waits on that guard, not on the child Option used for respawn.
7954            let mut exited_child = child.take().expect("exited reload child is still stored");
7955            #[cfg(test)]
7956            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7957                gate.reached.notify_one();
7958                gate.resume.notified().await;
7959            }
7960            let result = handle_reload_child_registration_failure(
7961                spec,
7962                runtime,
7963                registry,
7964                process_liveness,
7965                snapshot,
7966                child,
7967                ReloadRegistrationFailure {
7968                    exit_report: registration_failure_exit_report(exit_report),
7969                    reason: exited_child
7970                        .spawn_failure
7971                        .clone()
7972                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7973                },
7974            )
7975            .await;
7976            exited_child.release_roster();
7977            result
7978        }
7979        RegistrationWaitOutcome::TimedOut => {
7980            let mut timed_out_child = child
7981                .take()
7982                .expect("timed-out reload child is still running");
7983            timed_out_child
7984                .start_kill()
7985                .map_err(|source| SuperviseError::Kill {
7986                    module_id: spec.module_id.clone(),
7987                    source,
7988                })?;
7989            let status = timed_out_child
7990                .wait()
7991                .await
7992                .map_err(|source| SuperviseError::Wait {
7993                    module_id: spec.module_id.clone(),
7994                    source,
7995                })?;
7996            timed_out_child.drain_stderr(&spec.module_id).await;
7997            handle_reload_child_registration_failure(
7998                spec,
7999                runtime,
8000                registry,
8001                process_liveness,
8002                snapshot,
8003                child,
8004                ReloadRegistrationFailure {
8005                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8006                        snapshot,
8007                        &timed_out_child,
8008                        &status,
8009                    )),
8010                    reason: format!(
8011                        "new child did not register within {:?}",
8012                        REGISTRY_RELEASE_TIMEOUT
8013                    ),
8014                },
8015            )
8016            .await
8017        }
8018    }
8019}
8020
8021#[allow(clippy::too_many_arguments)]
8022async fn set_child_enabled(
8023    spec: &ModuleSpec,
8024    runtime: &SupervisorRuntimeConfig,
8025    registry: &Registry,
8026    process_liveness: &SupervisorProcessLiveness,
8027    snapshot: &SharedSnapshot,
8028    child: &mut Option<SupervisedChild>,
8029    enabled: bool,
8030    failure: &FailureContext,
8031) -> Result<bool, SuperviseError> {
8032    let (current_enabled, current_state, respawn_pending) = {
8033        let state = lock_snapshot(snapshot)?;
8034        (state.enabled, state.state, state.respawn_pending)
8035    };
8036    // `start` (enable on an already-enabled module) heals TERMINAL states instead
8037    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
8038    // clean (Stopped) has no live process and no other in-band recovery — the
8039    // operator's start is the explicit recovery act and resets the budget. Without
8040    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
8041    // which the 2026-07-14 aft outage proved is a trap when the failed module is
8042    // the one providing every agent's shell.
8043    let revive_terminal = enabled
8044        && current_enabled
8045        && child.is_none()
8046        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8047            || (current_state == ModuleState::Restarting && !respawn_pending));
8048    if current_enabled == enabled && !revive_terminal {
8049        return Ok(false);
8050    }
8051
8052    if enabled {
8053        update_snapshot(snapshot, Some(&spec.module_id), |state| {
8054            state.enabled = true;
8055            state.state = ModuleState::Starting;
8056            clear_current_process_facts(state);
8057        })?;
8058        #[cfg(test)]
8059        if runtime.test_seed_stale_facts_before_enable_spawn {
8060            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8061                state.process_alive = true;
8062                state.pid = Some(41);
8063                state.spawned_at_ms = Some(42);
8064                state.spawned_from = Some(PathBuf::from("/spawned/module"));
8065                state.spawned_file_identity = Some(SpawnedFileIdentity {
8066                    device: 43,
8067                    inode: 44,
8068                    #[cfg(windows)]
8069                    windows_image: None,
8070                });
8071            })?;
8072        }
8073        release_dead_registration(
8074            registry,
8075            runtime.forwarding.as_deref(),
8076            snapshot,
8077            &spec.module_id,
8078            failure,
8079        )
8080        .await?;
8081        reset_restart_count(snapshot, &spec.module_id)?;
8082        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8083        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8084            Ok(next_child) => next_child,
8085            Err(err) => {
8086                fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8087                process_liveness.untrack_if_current(&spec.module_id, snapshot);
8088                return Err(err);
8089            }
8090        };
8091        *child = Some(next_child);
8092        debug!(module_id = %spec.module_id, "supervised module enabled");
8093        Ok(true)
8094    } else {
8095        let stop_notice = begin_forwarding_drain_if_configured(
8096            spec,
8097            runtime,
8098            registry,
8099            snapshot,
8100            Some(false),
8101            RouteCloseReason::Disable,
8102        )
8103        .await?;
8104        drain_optional_child(
8105            &spec.module_id,
8106            spec.protocol,
8107            stop_notice,
8108            registry,
8109            runtime.forwarding.as_deref(),
8110            snapshot,
8111            &runtime.terminal_ring,
8112            &runtime.spawn_events,
8113            child,
8114            runtime.drain_timeout,
8115            ModuleState::Disabled,
8116            Some(false),
8117            failure,
8118        )
8119        .await?;
8120        debug!(module_id = %spec.module_id, "supervised module disabled");
8121        Ok(true)
8122    }
8123}
8124
8125#[allow(clippy::too_many_arguments)]
8126async fn on_child_exit(
8127    spec: &ModuleSpec,
8128    policy: RestartPolicy,
8129    registry: &Registry,
8130    snapshot: &SharedSnapshot,
8131    terminal_ring: &Arc<Mutex<TerminalRing>>,
8132    spawn_events: &SpawnEventFeed,
8133    roster: &ChildRoster,
8134    exit_report: ExitReport,
8135    daemon_initiated: bool,
8136) -> NextAction {
8137    // Once the daemon has begun shutting down, no exit is a crash to recover
8138    // from: the module is exiting because the daemon is going away (EOF on its
8139    // connection, or a service manager signalling the whole cgroup). Record it
8140    // as such and never schedule a respawn, which would only start a process
8141    // for the shutdown to end again.
8142    if roster.is_closed() {
8143        return on_child_exit_during_daemon_shutdown(
8144            spec,
8145            registry,
8146            snapshot,
8147            terminal_ring,
8148            spawn_events,
8149            exit_report,
8150        )
8151        .await;
8152    }
8153    // Deliberate stops the supervisor asks for (operator stop, disable,
8154    // restart, reload, swap, a health restart, a drain that runs out of budget)
8155    // takes the child out of the supervise loop and reaps it in
8156    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
8157    // that reaches this point is either unrequested or a first-HELLO timeout,
8158    // which is deliberately classified as a crash even after a clean TERM exit.
8159    //
8160    // For a subc-wire module a clean exit is still a stop: those modules are
8161    // written to re-raise SIGTERM, so a stray outside signal already reads as a
8162    // crash, and exiting 0 is a deliberate choice the module made. A
8163    // `protocol: "none"` module is a stock program we cannot change, and many
8164    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
8165    // stop would leave the module down for good after any stray signal, so it
8166    // goes through the crash path instead: it spends restart budget, respawns
8167    // with the crash backoff, and ends `failed` when the budget runs out.
8168    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8169        && running_protocol(spec, snapshot) == ModuleProtocol::None;
8170    match exit_report.kind {
8171        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8172            info!(
8173                module_id = %spec.module_id,
8174                exit_code = ?exit_report.code,
8175                exit_signal = ?exit_report.signal,
8176                "supervised module exited cleanly"
8177            );
8178            if let Err(err) = publish_after_terminal_record(
8179                &spec.module_id,
8180                || {
8181                    record_terminal(
8182                        &spec.module_id,
8183                        terminal_ring,
8184                        spawn_events,
8185                        &exit_report,
8186                        TerminalDisposition::Stopped,
8187                    )
8188                },
8189                || {
8190                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8191                        state.state = ModuleState::Stopped;
8192                        clear_current_process_facts(state);
8193                        state.last_exit = Some(exit_report.clone());
8194                    })
8195                },
8196            ) {
8197                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8198            }
8199            let registration_released = match wait_for_registration_release(
8200                registry,
8201                &spec.module_id,
8202                REGISTRY_RELEASE_TIMEOUT,
8203            )
8204            .await
8205            {
8206                Ok(()) => true,
8207                Err(err) => {
8208                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8209                    false
8210                }
8211            };
8212            NextAction::Stop {
8213                registration_released,
8214            }
8215        }
8216        ExitKind::Clean | ExitKind::Crash => {
8217            if unrequested_clean_exit_of_protocol_none {
8218                warn!(
8219                    module_id = %spec.module_id,
8220                    exit_code = ?exit_report.code,
8221                    exit_signal = ?exit_report.signal,
8222                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
8223                );
8224            } else {
8225                warn!(
8226                    module_id = %spec.module_id,
8227                    exit_code = ?exit_report.code,
8228                    exit_signal = ?exit_report.signal,
8229                    "supervised module exited abnormally (crash)"
8230                );
8231            }
8232            let mut restart_schedule = None;
8233            let mut disposition = TerminalDisposition::Disabled;
8234            let mut final_state = ModuleState::Disabled;
8235            // Set only when the budget is what stopped the module, so the
8236            // terminal record says which limit was hit rather than leaving
8237            // `failed` to be read as "crashed once, badly".
8238            let mut disposition_detail = lock_snapshot(snapshot)
8239                .ok()
8240                .and_then(|mut state| state.spawn_failure.take());
8241            let now = Instant::now();
8242            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8243                if state.enabled {
8244                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
8245                        // Reserving the last slot exhausts the budget for the
8246                        // replacement, not for this exit. Publish its scheduled
8247                        // recovery atomically with the reservation so connection
8248                        // cleanup sees it while terminal recording still holds
8249                        // the visible state at Running.
8250                        state.respawn_pending = true;
8251                        final_state = ModuleState::Restarting;
8252                        restart_schedule = Some(schedule);
8253                        disposition = TerminalDisposition::Restarting;
8254                    } else {
8255                        final_state = ModuleState::Failed;
8256                        disposition = TerminalDisposition::Failed;
8257                        let budget = policy.budget_exhausted_detail();
8258                        disposition_detail =
8259                            Some(disposition_detail.take().map_or_else(
8260                                || budget.clone(),
8261                                |cause| format!("{cause}; {budget}"),
8262                            ));
8263                    }
8264                } else {
8265                    disposition = TerminalDisposition::Disabled;
8266                }
8267            }) {
8268                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8269                return NextAction::Stop {
8270                    registration_released: false,
8271                };
8272            }
8273            if disposition == TerminalDisposition::Failed {
8274                // The window is in the message, not only in the fields: this line
8275                // is read in a scrollback where a bare `max_restarts=3` reads as a
8276                // lifetime cap and sends the operator looking for three crashes
8277                // that never happened together.
8278                error!(
8279                    module_id = %spec.module_id,
8280                    max_restarts = policy.max_restarts,
8281                    window_secs = policy.window.as_secs(),
8282                    "module stopped: {}",
8283                    policy.budget_exhausted_detail()
8284                );
8285            }
8286            if let Err(err) = publish_after_terminal_record(
8287                &spec.module_id,
8288                || {
8289                    record_terminal_with_detail(
8290                        &spec.module_id,
8291                        terminal_ring,
8292                        spawn_events,
8293                        &exit_report,
8294                        disposition,
8295                        disposition_detail,
8296                    )
8297                },
8298                || {
8299                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8300                        state.state = final_state;
8301                        clear_current_process_facts(state);
8302                        if final_state == ModuleState::Failed
8303                            && daemon_initiated
8304                            && !roster.is_closed()
8305                        {
8306                            state.failed_retry.schedule(&spec.module_id);
8307                        }
8308                        state.last_exit = Some(exit_report.clone());
8309                    })
8310                },
8311            ) {
8312                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8313            }
8314
8315            if let Some(schedule) = restart_schedule {
8316                NextAction::Restart {
8317                    schedule: Some(schedule),
8318                }
8319            } else {
8320                let registration_released = match wait_for_registration_release(
8321                    registry,
8322                    &spec.module_id,
8323                    REGISTRY_RELEASE_TIMEOUT,
8324                )
8325                .await
8326                {
8327                    Ok(()) => true,
8328                    Err(err) => {
8329                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8330                        false
8331                    }
8332                };
8333                NextAction::Stop {
8334                    registration_released,
8335                }
8336            }
8337        }
8338        ExitKind::DeliberateSeverance => {
8339            warn!(
8340                module_id = %spec.module_id,
8341                exit_code = ?exit_report.code,
8342                exit_signal = ?exit_report.signal,
8343                "supervised module exited after deliberate connection severance"
8344            );
8345            let should_restart = match lock_snapshot(snapshot) {
8346                Ok(state) => state.enabled,
8347                Err(err) => {
8348                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8349                    return NextAction::Stop {
8350                        registration_released: false,
8351                    };
8352                }
8353            };
8354            let final_state = if should_restart {
8355                ModuleState::Restarting
8356            } else {
8357                ModuleState::Disabled
8358            };
8359            if let Err(err) = publish_after_terminal_record(
8360                &spec.module_id,
8361                || {
8362                    record_terminal(
8363                        &spec.module_id,
8364                        terminal_ring,
8365                        spawn_events,
8366                        &exit_report,
8367                        terminal_disposition(final_state),
8368                    )
8369                },
8370                || {
8371                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8372                        state.state = final_state;
8373                        clear_current_process_facts(state);
8374                        state.last_exit = Some(exit_report.clone());
8375                        state.lifetime_restarts += 1;
8376                    })
8377                },
8378            ) {
8379                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8380                return NextAction::Stop {
8381                    registration_released: false,
8382                };
8383            }
8384
8385            if should_restart {
8386                NextAction::Restart { schedule: None }
8387            } else {
8388                let registration_released = match wait_for_registration_release(
8389                    registry,
8390                    &spec.module_id,
8391                    REGISTRY_RELEASE_TIMEOUT,
8392                )
8393                .await
8394                {
8395                    Ok(()) => true,
8396                    Err(err) => {
8397                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8398                        false
8399                    }
8400                };
8401                NextAction::Stop {
8402                    registration_released,
8403                }
8404            }
8405        }
8406    }
8407}
8408
8409async fn on_child_exit_during_daemon_shutdown(
8410    spec: &ModuleSpec,
8411    registry: &Registry,
8412    snapshot: &SharedSnapshot,
8413    terminal_ring: &Arc<Mutex<TerminalRing>>,
8414    spawn_events: &SpawnEventFeed,
8415    exit_report: ExitReport,
8416) -> NextAction {
8417    info!(
8418        module_id = %spec.module_id,
8419        exit_code = ?exit_report.code,
8420        exit_signal = ?exit_report.signal,
8421        exit_kind = ?exit_report.kind,
8422        "supervised module exited during daemon shutdown; not restarting it"
8423    );
8424    if let Err(err) = publish_after_terminal_record(
8425        &spec.module_id,
8426        || {
8427            record_terminal(
8428                &spec.module_id,
8429                terminal_ring,
8430                spawn_events,
8431                &exit_report,
8432                TerminalDisposition::DaemonShutdown,
8433            )
8434        },
8435        || {
8436            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8437                state.state = ModuleState::Stopped;
8438                clear_current_process_facts(state);
8439                state.last_exit = Some(exit_report.clone());
8440            })
8441        },
8442    ) {
8443        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8444    }
8445    let registration_released =
8446        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8447            .await
8448            .is_ok();
8449    NextAction::Stop {
8450        registration_released,
8451    }
8452}
8453
8454// Keep the status lock out of terminal I/O. Readers may observe the previous
8455// state while recording is in progress, but never a completed exit without its
8456// history. Journal errors are logged by the journal and retained in the ring;
8457// a recording panic still publishes the final state before resuming the unwind.
8458fn publish_after_terminal_record<T>(
8459    module_id: &str,
8460    record: impl FnOnce(),
8461    publish: impl FnOnce() -> T,
8462) -> T {
8463    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8464    let published = publish();
8465    if let Err(panic) = recorded {
8466        error!(
8467            module_id,
8468            "terminal recording panicked; published final state before resuming unwind"
8469        );
8470        std::panic::resume_unwind(panic);
8471    }
8472    published
8473}
8474
8475fn fail_after_wait_error(
8476    module_id: &str,
8477    snapshot: &SharedSnapshot,
8478    terminal_ring: &Arc<Mutex<TerminalRing>>,
8479    spawn_events: &SpawnEventFeed,
8480    detail: String,
8481    failure: &FailureContext,
8482) {
8483    let report = wait_error_exit_report();
8484    publish_after_terminal_record(
8485        module_id,
8486        || {
8487            record_terminal_with_detail(
8488                module_id,
8489                terminal_ring,
8490                spawn_events,
8491                &report,
8492                TerminalDisposition::Failed,
8493                Some(detail),
8494            )
8495        },
8496        || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8497    );
8498}
8499
8500#[cfg(test)]
8501fn record_wait_error_terminal(
8502    module_id: &str,
8503    terminal_ring: &Arc<Mutex<TerminalRing>>,
8504    spawn_events: &SpawnEventFeed,
8505) {
8506    record_terminal(
8507        module_id,
8508        terminal_ring,
8509        spawn_events,
8510        &wait_error_exit_report(),
8511        TerminalDisposition::Failed,
8512    );
8513}
8514
8515fn record_terminal(
8516    module_id: &str,
8517    terminal_ring: &Arc<Mutex<TerminalRing>>,
8518    spawn_events: &SpawnEventFeed,
8519    exit_report: &ExitReport,
8520    disposition: TerminalDisposition,
8521) {
8522    record_terminal_with_detail(
8523        module_id,
8524        terminal_ring,
8525        spawn_events,
8526        exit_report,
8527        disposition,
8528        None,
8529    );
8530}
8531
8532/// The ring lock is held only to capture the read (see
8533/// `TerminalJournal::capture_read`), so this module's exits keep recording
8534/// while the journal files are read. Blocking: it reads files.
8535fn durable_terminal_history_of(
8536    terminal_ring: &Mutex<TerminalRing>,
8537    module_id: &str,
8538) -> subc_control::TerminalHistory {
8539    let read = terminal_ring
8540        .lock()
8541        .unwrap_or_else(|p| p.into_inner())
8542        .capture_durable_history();
8543    read.read(module_id)
8544}
8545
8546fn record_terminal_with_detail(
8547    module_id: &str,
8548    terminal_ring: &Arc<Mutex<TerminalRing>>,
8549    spawn_events: &SpawnEventFeed,
8550    exit_report: &ExitReport,
8551    disposition: TerminalDisposition,
8552    disposition_detail: Option<String>,
8553) {
8554    #[cfg(test)]
8555    terminal_history_tests::before_terminal_record(spawn_events);
8556    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8557    let record = TerminalRecord {
8558        exit_code: exit_report.code,
8559        exit_signal: exit_report.signal,
8560        at_ms: exit_report.at_ms,
8561        disposition,
8562        exit_kind: exit_report.kind.into(),
8563        disposition_detail,
8564    };
8565    terminal_ring
8566        .lock()
8567        .unwrap_or_else(|poisoned| poisoned.into_inner())
8568        .record_exit(module_id, record);
8569}
8570
8571fn untrack_if_registration_released(
8572    process_liveness: &SupervisorProcessLiveness,
8573    registry: &Registry,
8574    module_id: &str,
8575    snapshot: &SharedSnapshot,
8576) {
8577    match registry.get_module(module_id) {
8578        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8579        Ok(Some(_)) => {}
8580        Err(err) => {
8581            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8582        }
8583    }
8584}
8585
8586/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8587/// then apply the module's configured entries minus daemon-private capture keys.
8588///
8589/// Separated from `spawn_child` only so it can be asserted without spawning a
8590/// process — a duplicate of this logic in a test would pass while the real one
8591/// drifted, which is the defect class this function exists to avoid.
8592/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8593/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8594/// either and the argument would stop a stock binary from starting at all.
8595/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8596///
8597/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8598/// through [`apply_wire_spawn_args_for_role`].
8599#[cfg(test)]
8600fn apply_wire_spawn_args(
8601    command: &mut Command,
8602    spec: &ModuleSpec,
8603    connection_file_path: Option<&std::path::Path>,
8604    handle: Option<&SupervisorHandle>,
8605) -> Result<Option<NonceHandoff>, SuperviseError> {
8606    apply_wire_spawn_args_for_role(
8607        command,
8608        spec,
8609        connection_file_path,
8610        handle,
8611        SpawnRole::Plain,
8612    )
8613}
8614
8615/// The read end of a spawn's launch-nonce pipe, prepared by
8616/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8617/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8618/// server before spawn and authenticates its client after the child exists.
8619#[cfg(unix)]
8620type NonceHandoff = subc_os::LaunchNonceHandoff;
8621#[cfg(windows)]
8622type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8623#[cfg(not(any(unix, windows)))]
8624type NonceHandoff = std::convert::Infallible;
8625
8626/// Create the launch nonce that a new module process must present when it
8627/// registers or opens a consumer connection, including a replacement process
8628/// started alongside a still-serving module.
8629///
8630/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8631/// a separate candidate token so the still-serving incumbent and its consumers
8632/// keep their nonce. Both records are installed before the process exists, so
8633/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8634///
8635/// On Unix the nonce is delivered only through a pipe. It is written into
8636/// a pipe whose read end the child gets as descriptor 3, named by
8637/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8638/// process of the same user cannot read it with `ps eww`. That handoff is
8639/// returned rather than installed here, because installing it replaces
8640/// whatever the child has at descriptor 3 and so must be the last pre-exec
8641/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8642/// the Linux callback that writes the process into its cgroup. Only after that
8643/// does `handoff.install_last` register the descriptor-3 replacement, so it
8644/// cannot close a descriptor the cgroup callback still needs.
8645/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8646fn apply_wire_spawn_args_for_role(
8647    command: &mut Command,
8648    spec: &ModuleSpec,
8649    connection_file_path: Option<&std::path::Path>,
8650    handle: Option<&SupervisorHandle>,
8651    role: SpawnRole,
8652) -> Result<Option<NonceHandoff>, SuperviseError> {
8653    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8654    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8655    // included: a daemon started from a module's process tree inherits it,
8656    // and passing it on would point the child at a descriptor it does not
8657    // have.
8658    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8659    #[cfg(windows)]
8660    {
8661        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8662        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8663    }
8664    // Remove inherited or configured copies too: withholding must mean absent.
8665    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8666    if spec.protocol == ModuleProtocol::None {
8667        return Ok(None);
8668    }
8669    if let Some(connection_file_path) = connection_file_path {
8670        command.arg(SUBC_ARG).arg(connection_file_path);
8671    }
8672
8673    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8674    // route.open attestation. Reserved modules additionally use the same nonce
8675    // for HELLO id-squatting protection. A respawn rotates both records.
8676    let nonce = generate_launch_nonce()?;
8677    if let Some(handle) = handle {
8678        match role {
8679            SpawnRole::Plain => {
8680                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8681                if spec.reserved {
8682                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8683                }
8684            }
8685            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8686        }
8687    }
8688    #[cfg(unix)]
8689    let handoff = {
8690        let handoff =
8691            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8692                program: spec.program.clone(),
8693                source,
8694                cgroup_path: None,
8695            })?;
8696        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8697        Some(handoff)
8698    };
8699    #[cfg(windows)]
8700    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8701    #[cfg(not(any(unix, windows)))]
8702    let handoff = None;
8703    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8704    // until the daemon's per-spawn delivery records show that every running
8705    // Windows module has read its secret from the named pipe.
8706    #[cfg(not(unix))]
8707    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8708    Ok(handoff)
8709}
8710
8711fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8712    command.env_remove(CK_LOG_ENV);
8713    // The spawn role is the supervisor's to set, and only on a swap candidate
8714    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8715    // it, is what makes it absent on a plain spawn: the daemon's own
8716    // environment could carry it, and so could a spec built outside daemon
8717    // config (config refuses it as an `env` key). A module reading it on a
8718    // plain restart would pick the long swap budget and leave callers waiting.
8719    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8720    for (key, value) in &spec.env {
8721        // cortexkit-log currently exposes retention only as a Rust struct, not
8722        // environment names. These values are daemon-private sink metadata and
8723        // must never become a public child-process contract by being inherited.
8724        if matches!(
8725            key.as_str(),
8726            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8727        ) || key == SUBC_SPAWN_ROLE_ENV
8728        {
8729            continue;
8730        }
8731        command.env(key, value);
8732    }
8733}
8734
8735#[cfg(windows)]
8736fn prepare_windows_nonce_pipe(
8737    command: &mut Command,
8738    spec: &ModuleSpec,
8739    nonce: &str,
8740) -> Option<NonceHandoff> {
8741    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8742        Ok(handoff) => {
8743            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8744            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8745            // still register until every module reads the pipe. A reader without
8746            // this variable set to env returns the pipe error instead of using
8747            // an inherited or configured environment copy.
8748            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8749            Some(handoff)
8750        }
8751        Err(error) => {
8752            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8753            None
8754        }
8755    }
8756}
8757
8758#[cfg(windows)]
8759fn start_windows_nonce_pipe(
8760    handoff: Option<NonceHandoff>,
8761    pid: u32,
8762    spec: &ModuleSpec,
8763) -> Option<subc_os::LaunchNoncePipeDelivery> {
8764    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8765    // same maximum wait used for registration during a reload. Older modules
8766    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8767    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8768        Ok(delivery) => Some(delivery),
8769        Err(error) => {
8770            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8771            None
8772        }
8773    })
8774}
8775
8776fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8777    #[cfg(windows)]
8778    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8779        return Some(
8780            if snapshot
8781                .pipe_consumed
8782                .as_ref()
8783                .is_some_and(|consumed| consumed.get().is_some())
8784            {
8785                "pipe"
8786            } else {
8787                "env"
8788            }
8789            .to_owned(),
8790        );
8791    }
8792    let _ = snapshot;
8793    None
8794}
8795
8796/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8797/// of a blue/green swap.
8798#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8799enum SpawnRole {
8800    Plain,
8801    SwapCandidate,
8802}
8803
8804/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8805/// `apply_child_env` has already removed the variable for every spawn.
8806fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8807    if role == SpawnRole::SwapCandidate {
8808        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8809    }
8810}
8811
8812fn previous_instance_error(
8813    spec: &ModuleSpec,
8814    pid: u32,
8815    reason: crate::live_children::RetainedReason,
8816) -> SuperviseError {
8817    SuperviseError::Spawn {
8818        program: spec.program.clone(),
8819        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8820        cgroup_path: None,
8821    }
8822}
8823
8824fn spawn_child(
8825    spec: &ModuleSpec,
8826    connection_file_path: Option<&std::path::Path>,
8827    handle: Option<&SupervisorHandle>,
8828    ring: &Arc<Mutex<StderrRing>>,
8829    capture_logs_dir: Option<&std::path::Path>,
8830    roster: &ChildRoster,
8831    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8832) -> Result<SupervisedChild, SuperviseError> {
8833    spawn_child_in_slot(
8834        spec,
8835        connection_file_path,
8836        handle,
8837        ring,
8838        capture_logs_dir,
8839        roster,
8840        #[cfg(target_os = "linux")]
8841        cgroup_placement,
8842        SpawnRole::Plain,
8843        false,
8844    )
8845}
8846
8847/// Spawn one process of `spec` into a slot.
8848///
8849/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8850/// A swap candidate needs a different cgroup from the process it is replacing,
8851/// which is still alive: in the same cgroup the two would be one kill domain,
8852/// and killing a failed candidate could take the incumbent with it.
8853///
8854/// The stderr capture file is `<module_id>.stderr.log` for every process of
8855/// the module, whichever slot it is in, because that is the one file
8856/// `ck module logs` reads. During a swap's overlap both processes append to it;
8857/// the daemon writes whole lines, so the two interleave by line, which is also
8858/// the merged view an operator wants while a swap runs.
8859#[allow(clippy::too_many_arguments)]
8860fn spawn_child_in_slot(
8861    spec: &ModuleSpec,
8862    connection_file_path: Option<&std::path::Path>,
8863    handle: Option<&SupervisorHandle>,
8864    ring: &Arc<Mutex<StderrRing>>,
8865    capture_logs_dir: Option<&std::path::Path>,
8866    roster: &ChildRoster,
8867    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8868    role: SpawnRole,
8869    alternate_slot: bool,
8870) -> Result<SupervisedChild, SuperviseError> {
8871    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8872        return Err(previous_instance_error(spec, pid, reason));
8873    }
8874    if roster.is_closed() {
8875        return Err(SuperviseError::Spawn {
8876            program: spec.program.clone(),
8877            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8878            cgroup_path: None,
8879        });
8880    }
8881    #[cfg(target_os = "linux")]
8882    let cgroup_name = {
8883        // Slot names alone are not kill domains: a retired incumbent may still
8884        // be draining when a later enable/restart spawns into the same slot.
8885        // Decimal entropy keeps the suffix unambiguous; Placement performs
8886        // the module-id escaping and constructs the filesystem path.
8887        if cgroup_placement.is_none() {
8888            swap::cgroup_name(&spec.module_id, alternate_slot)
8889        } else {
8890            let nonce = generate_launch_nonce()?;
8891            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8892            // Leave room for byte escaping and the suffix under NAME_MAX. The
8893            // label is only for humans; the nonce identifies the kill domain.
8894            let mut end = spec.module_id.len().min(64);
8895            while !spec.module_id.is_char_boundary(end) {
8896                end -= 1;
8897            }
8898            format!(
8899                "{}_{suffix}",
8900                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8901            )
8902        }
8903    };
8904    #[cfg(not(target_os = "linux"))]
8905    let _ = alternate_slot;
8906    #[cfg(target_os = "macos")]
8907    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8908    #[cfg(windows)]
8909    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8910        program: spec.program.clone(),
8911        source,
8912        cgroup_path: None,
8913    })?;
8914    #[cfg(windows)]
8915    let mut command = Command::new(image_capture.path());
8916    #[cfg(not(any(target_os = "macos", windows)))]
8917    let mut command = Command::new(&spec.program);
8918    command.args(&spec.args);
8919    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8920    // that is the whole of the intent, so remove that one key rather than the
8921    // environment.
8922    //
8923    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8924    // and took the POSIX environment with it. Modules spawned that way had no
8925    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8926    // logging:
8927    //
8928    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8929    //     both unset it fell back to the temp dir alone and `ck` could not find
8930    //     a daemon running on the same machine from inside any module's process
8931    //     tree — reporting a path the file has never lived at, which reads as
8932    //     "the daemon did not write its file".
8933    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8934    //     the RELATIVE `.local/share`, so a module deriving its own store path
8935    //     resolved it against its own CWD. That is the store-fragmentation
8936    //     defect the daemon already refuses in config (`parse_doc` rejects a
8937    //     relative `storage.data_home`) arriving by derivation instead.
8938    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8939    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8940    //     quietly rather than erroring.
8941    //
8942    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8943    // offered one candidate under /tmp while the file sat in /run/user/1000.
8944    //
8945    // A configured module is unaffected either way: `module_spec()` puts the
8946    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8947    // wins over anything ambient.
8948    apply_child_env(&mut command, spec);
8949    apply_spawn_role(&mut command, role);
8950    let nonce_handoff =
8951        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8952
8953    #[cfg(target_os = "linux")]
8954    let cgroup_path = cgroup_placement
8955        .map(|placement| placement.module_path(&cgroup_name))
8956        .transpose()
8957        .map_err(|source| SuperviseError::Cgroup {
8958            module_id: spec.module_id.clone(),
8959            source,
8960        })?;
8961    #[cfg(not(target_os = "linux"))]
8962    let cgroup_path: Option<PathBuf> = None;
8963    #[cfg(target_os = "linux")]
8964    if let Some(path) = &cgroup_path {
8965        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8966            if let Some(placement) = cgroup_placement {
8967                remove_module_cgroup(placement, &cgroup_name);
8968            }
8969            return Err(error);
8970        }
8971    }
8972
8973    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8974        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8975        match ChildOutputSink::open(&path, capture_retention(spec)) {
8976            Ok(sink) => sink,
8977            Err(error) => {
8978                warn!(
8979                    module_id = %spec.module_id,
8980                    path = %path.display(),
8981                    error = %error,
8982                    "could not open child output capture file; forwarding to stderr"
8983                );
8984                ChildOutputSink::Stderr
8985            }
8986        }
8987    } else {
8988        ChildOutputSink::Stderr
8989    };
8990
8991    command.stdout(Stdio::piped());
8992    command.stderr(Stdio::piped());
8993    command.kill_on_drop(true);
8994    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8995    // before exec). In the daemon's group, a service manager that kills the
8996    // job's process group when the daemon exits (launchd's default) killed
8997    // every module at the same moment its control connection closed, so no
8998    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8999    // module is reached only by the daemon: the EOF it sees when its
9000    // connection closes, and the bounded stop in `child_roster` for anything
9001    // still running after that. On Linux this composes with the cgroup
9002    // placement above: that is a pre_exec write to cgroup.procs, std performs
9003    // setpgid in the child before running pre_exec callbacks, and the two
9004    // change independent process attributes.
9005    //
9006    // stdin is /dev/null because a process outside the terminal's foreground
9007    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
9008    // by hand would otherwise hand down. Under a service manager stdin is
9009    // already /dev/null.
9010    #[cfg(unix)]
9011    command.process_group(0);
9012    command.stdin(Stdio::null());
9013    // The LAST pre-exec step, after the cgroup placement above: installing the
9014    // nonce at descriptor 3 replaces whatever the child had there, which could
9015    // be the descriptor an earlier step writes through.
9016    #[cfg(unix)]
9017    if let Some(handoff) = nonce_handoff {
9018        handoff.install_last(command.as_std_mut());
9019    }
9020    #[cfg(not(any(unix, windows)))]
9021    let _ = nonce_handoff;
9022
9023    // Containment, step 1 of 3 (issue #109): create the child suspended so it
9024    // cannot run a single instruction -- and therefore cannot spawn a
9025    // grandchild -- before it is in the job. See `contain_spawned_child` for the
9026    // other two steps and why the window matters.
9027    #[cfg(windows)]
9028    {
9029        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9030            program: spec.program.clone(),
9031            source,
9032            cgroup_path: None,
9033        })?;
9034        subc_jobobject::suspend_on_create_async(&mut command);
9035    }
9036    // Timestamp before the spawn call: on Unix the child can serve before that
9037    // call even returns. This conservative bound never labels an older child
9038    // as a replacement for a request received during process creation.
9039    let spawned_at = Instant::now();
9040    let mut child = match command.spawn() {
9041        Ok(child) => child,
9042        Err(source) => {
9043            #[cfg(target_os = "linux")]
9044            if let Some(placement) = cgroup_placement {
9045                remove_module_cgroup(placement, &cgroup_name);
9046            }
9047            return Err(SuperviseError::Spawn {
9048                program: spec.program.clone(),
9049                source,
9050                cgroup_path,
9051            });
9052        }
9053    };
9054    #[cfg(test)]
9055    spawn_record_tests::after_create(&child, spec);
9056    // The parent must close its writer now: the acknowledgement pipe reports EOF
9057    // only when every writer is gone, and the child's copy closes when the
9058    // trampoline replaces itself with the module. Command holds only an integer
9059    // in its pre_exec callback, not another writer.
9060    #[cfg(target_os = "macos")]
9061    drop(exec_ack);
9062
9063    // Bind while the child is still suspended, before job assignment resumes it.
9064    // Keeping the executable handle open across CreateProcessW denies rename
9065    // and deletion, so the file cannot be exchanged during spawn. Bind that
9066    // file ID to the process handle returned by spawn, never a reopened PID.
9067    #[cfg(windows)]
9068    let windows_image = match image_capture.bind_tokio(&child) {
9069        Ok(image) => Arc::new(image),
9070        Err(source) => {
9071            kill_and_reap_rejected_child(&mut child);
9072            return Err(SuperviseError::Spawn {
9073                program: spec.program.clone(),
9074                source,
9075                cgroup_path: None,
9076            });
9077        }
9078    };
9079    // Assign the suspended child to its module job; resume only after its
9080    // process identity has been durably recorded for orphan recovery.
9081    #[cfg(windows)]
9082    let job = contain_spawned_child(&mut child, spec)?;
9083    let spawned_at_ms = unix_ms_now();
9084    #[cfg(not(windows))]
9085    let spawned_from = spec.program.clone();
9086    #[cfg(windows)]
9087    let spawned_from = windows_image.path().to_path_buf();
9088    #[cfg(not(windows))]
9089    let spawned_file_identity = spawned_file_identity(&spawned_from);
9090    #[cfg(windows)]
9091    let spawned_file_identity = Some(SpawnedFileIdentity {
9092        device: 0,
9093        inode: 0,
9094        windows_image: Some(Arc::clone(&windows_image)),
9095    });
9096    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9097        program: spec.program.clone(),
9098        source: io::Error::other("spawned child exposed no live pid"),
9099        cgroup_path: cgroup_path.clone(),
9100    })?;
9101    #[cfg(windows)]
9102    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9103    #[cfg(not(windows))]
9104    let process_start_time = crate::provenance::process_start_time(pid);
9105    #[cfg(windows)]
9106    let process_start_time = Some(windows_image.start_time());
9107    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9108    #[cfg(all(test, target_os = "macos"))]
9109    privacy_exec_boundary_tests::before_image_sample(spec, pid);
9110    // Record the committed kernel image, not the configured pathname: PATH
9111    // lookup and shebang interpretation can select a different executable.
9112    // Linux's posix_spawn can return before that image is installed, so the
9113    // observer also waits for the kernel's first-exec flag to clear.
9114    #[cfg(not(windows))]
9115    let recorded_image = observe_spawned_image(pid);
9116    // spawn() confirms only the first exec, into the trampoline. Never persist
9117    // the trampoline image; the asynchronous acknowledgement publishes the
9118    // module image once the trampoline has replaced itself with the module.
9119    #[cfg(target_os = "macos")]
9120    let recorded_image = if privacy_exec.is_some() {
9121        None
9122    } else {
9123        recorded_image
9124    };
9125    #[cfg(target_os = "linux")]
9126    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9127    #[cfg(not(target_os = "linux"))]
9128    let recorded_cgroup_name = None;
9129    let roster_guard = roster.admit(
9130        spec.module_id.clone(),
9131        pid,
9132        spec.protocol,
9133        process_start_time,
9134        crate::child_roster::RecordedIdentity {
9135            #[cfg(not(windows))]
9136            start_time: recorded_image.map(|image| image.start_time),
9137            #[cfg(windows)]
9138            start_time: process_start_time,
9139            #[cfg(not(windows))]
9140            executable: recorded_image
9141                .and_then(|image| image.executable)
9142                .map(crate::live_children::ExecutableIdentity::from),
9143            #[cfg(windows)]
9144            executable: Some(crate::live_children::ExecutableIdentity::from(
9145                windows_image.identity(),
9146            )),
9147            cgroup_name: recorded_cgroup_name,
9148            #[cfg(target_os = "linux")]
9149            cgroup_placement: cgroup_placement.cloned(),
9150        },
9151    );
9152    let roster_guard = match roster_guard {
9153        Ok(guard) => guard,
9154        Err(failure) => {
9155            return failure.with_cleanup(|source| {
9156                #[cfg(windows)]
9157                if let Some(job) = &job {
9158                    let _ = job.terminate();
9159                }
9160                #[cfg(target_os = "linux")]
9161                kill_module_cgroup(cgroup_placement, &cgroup_name);
9162                kill_and_reap_rejected_child(&mut child);
9163                #[cfg(target_os = "linux")]
9164                if let Some(placement) = cgroup_placement {
9165                    remove_module_cgroup(placement, &cgroup_name);
9166                }
9167                Err(SuperviseError::Spawn {
9168                    program: spec.program.clone(),
9169                    source: io::Error::other(format!(
9170                        "could not durably record module child: {source}"
9171                    )),
9172                    cgroup_path,
9173                })
9174            });
9175        }
9176    };
9177    // The pre-spawn check can pass just before shutdown begins. Ownership is
9178    // published at the start of admission, even while a record write is queued.
9179    // If shutdown already saw an empty roster before that publication, this
9180    // check sees the roster closed and ends the child before starting a module.
9181    if roster.is_closed() {
9182        // This child was never admitted, so there is no module protocol shutdown to wait for.
9183        #[cfg(target_os = "linux")]
9184        kill_module_cgroup(cgroup_placement, &cgroup_name);
9185        if let Err(error) = child.start_kill() {
9186            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9187        }
9188        #[cfg(target_os = "linux")]
9189        if let Some(placement) = cgroup_placement {
9190            // This spawn was never admitted, so shutdown has no roster entry
9191            // to await. Do not detach its cleanup: the runtime could exit
9192            // before that task reaps the rejected child and removes its group.
9193            while matches!(child.try_wait(), Ok(None)) {
9194                std::thread::yield_now();
9195            }
9196            if matches!(
9197                subc_cgroup::kill_module(Some(placement), &cgroup_name),
9198                subc_cgroup::KillOutcome::Killed
9199            ) {
9200                if let Ok(path) = placement.module_path(&cgroup_name) {
9201                    while std::fs::read_to_string(path.join("cgroup.events"))
9202                        .ok()
9203                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9204                    {
9205                        std::thread::yield_now();
9206                    }
9207                }
9208            }
9209            remove_module_cgroup(placement, &cgroup_name);
9210        }
9211        drop(roster_guard);
9212        return Err(SuperviseError::Spawn {
9213            program: spec.program.clone(),
9214            source: io::Error::other(
9215                "the daemon began shutting down while this process was starting; ended it",
9216            ),
9217            cgroup_path,
9218        });
9219    }
9220
9221    #[cfg(windows)]
9222    if let Err(error) = resume_suspended_child(pid, spec) {
9223        kill_and_reap_rejected_child(&mut child);
9224        return Err(error);
9225    }
9226
9227    let stdout_pump = match child.stdout.take() {
9228        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9229        None => {
9230            warn!(
9231                module_id = %spec.module_id,
9232                "spawned child exposed no stdout pipe; file capture will be incomplete"
9233            );
9234            None
9235        }
9236    };
9237    let stderr_pump = match child.stderr.take() {
9238        Some(stderr) => {
9239            let generation = ring
9240                .lock()
9241                .unwrap_or_else(|poisoned| poisoned.into_inner())
9242                .begin_process();
9243            Some(StderrPump {
9244                task: tokio::spawn(pump_stderr_to(
9245                    stderr,
9246                    Arc::clone(ring),
9247                    generation,
9248                    output_sink,
9249                )),
9250                generation,
9251            })
9252        }
9253        None => {
9254            // Spawning succeeded but the pipe did not materialise. Recording it as
9255            // uncaptured keeps the tail honest: the alternative is an empty tail
9256            // that reads as a module which printed nothing.
9257            ring.lock()
9258                .unwrap_or_else(|poisoned| poisoned.into_inner())
9259                .mark_not_captured("stderr pipe was not available on spawn");
9260            warn!(
9261                module_id = %spec.module_id,
9262                "spawned child exposed no stderr pipe; tail will be unavailable"
9263            );
9264            None
9265        }
9266    };
9267
9268    let exec = Arc::new(crate::activity::ExecOutcome::new());
9269    #[cfg(target_os = "macos")]
9270    let pending_exec = privacy_exec.is_some();
9271    #[cfg(not(target_os = "macos"))]
9272    let pending_exec = false;
9273    if !pending_exec {
9274        exec.finish(crate::activity::ExecState::CommittedExec);
9275    }
9276    #[cfg(windows)]
9277    let job = job.map(|job| Arc::new(Mutex::new(job)));
9278    #[cfg(windows)]
9279    let subtree_target = job
9280        .as_ref()
9281        .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9282        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9283    #[cfg(target_os = "linux")]
9284    let subtree_target = cgroup_path
9285        .map(crate::subtree_resources::Target::Cgroup)
9286        .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9287    #[cfg(not(any(target_os = "linux", windows)))]
9288    let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9289    Ok(SupervisedChild {
9290        subtree_target,
9291        child,
9292        exec: crate::activity::ExecOwner(exec),
9293        #[cfg(windows)]
9294        nonce_delivery,
9295        protocol: spec.protocol,
9296        #[cfg(target_os = "linux")]
9297        module_id: cgroup_name,
9298        #[cfg(target_os = "linux")]
9299        cgroup_placement: cgroup_placement.cloned(),
9300        #[cfg(windows)]
9301        job,
9302        stdout_pump,
9303        stderr_pump,
9304        stderr_ring: Arc::clone(ring),
9305        spawned_at,
9306        registration_deadline: None,
9307        registration_watch: None,
9308        spawned_at_ms,
9309        spawned_from,
9310        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9311        spawned_file_identity,
9312        process_start_time,
9313        process_identity,
9314        pid,
9315        roster_guard: Some(roster_guard),
9316        #[cfg(target_os = "macos")]
9317        privacy_exec,
9318        #[cfg(target_os = "macos")]
9319        report_ready: Arc::new(OnceLock::new()),
9320        spawn_failure: None,
9321    })
9322}
9323
9324#[cfg(test)]
9325mod spawn_record_tests {
9326    use super::*;
9327
9328    thread_local! {
9329        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9330    }
9331
9332    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9333        LAST_PID.set(child.id());
9334        #[cfg(windows)]
9335        if let Some((_, address)) = _spec
9336            .env
9337            .iter()
9338            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9339        {
9340            use std::io::{Read, Write};
9341            let mut stream = std::net::TcpStream::connect(address).unwrap();
9342            stream
9343                .write_all(&child.id().unwrap().to_ne_bytes())
9344                .unwrap();
9345            let mut release = [0];
9346            stream.read_exact(&mut release).unwrap();
9347        }
9348    }
9349
9350    #[cfg(windows)]
9351    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9352        if spec
9353            .env
9354            .iter()
9355            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9356        {
9357            Err(io::Error::other("injected module job assignment failure"))
9358        } else {
9359            Ok(())
9360        }
9361    }
9362
9363    fn spec() -> ModuleSpec {
9364        ModuleSpec {
9365            module_id: "spawn-record-worker".into(),
9366            #[cfg(unix)]
9367            program: "/bin/sleep".into(),
9368            #[cfg(windows)]
9369            program: std::env::current_exe().unwrap(),
9370            #[cfg(unix)]
9371            args: vec!["60".into()],
9372            #[cfg(windows)]
9373            args: vec![
9374                "--exact".into(),
9375                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9376                "--nocapture".into(),
9377            ],
9378            env: vec![],
9379            reserved: false,
9380            reserved_prefixes: vec![],
9381            protocol: ModuleProtocol::None,
9382            overlap: Default::default(),
9383        }
9384    }
9385
9386    #[tokio::test]
9387    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9388        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9389        let supervisor =
9390            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9391                .with_live_children_record(home.join("live-children.json"));
9392        crate::live_children::fail_next_file_sync();
9393        let result = supervisor.spawn(spec());
9394        let failed = result.is_err();
9395        if let Ok(module) = result {
9396            module.stop().await.unwrap();
9397        }
9398        assert!(failed, "record failure must fail spawn");
9399        let pid = LAST_PID.get().expect("the real OS child was created");
9400        assert!(
9401            subc_os::Process::open(pid).unwrap().is_none(),
9402            "child {pid} survived rejected admission or was not reaped"
9403        );
9404    }
9405
9406    #[cfg(target_os = "macos")]
9407    #[test]
9408    #[ignore = "manual on-device latency measurement"]
9409    fn durable_record_write_latency_50() {
9410        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9411        let path = home.join("live-children.json");
9412        let children = vec![crate::live_children::LiveChild {
9413            module_id: "latency-worker".into(),
9414            pid: std::process::id(),
9415            protocol: ModuleProtocol::None,
9416            start_time: Some(1),
9417            executable: None,
9418            cgroup_name: None,
9419        }];
9420        let mut samples = Vec::new();
9421        for _ in 0..50 {
9422            let start = std::time::Instant::now();
9423            crate::live_children::write_record(&path, &children).unwrap();
9424            samples.push(start.elapsed());
9425        }
9426        samples.sort();
9427        eprintln!(
9428            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9429            (samples[24] + samples[25]) / 2,
9430            samples[47]
9431        );
9432    }
9433
9434    #[cfg(windows)]
9435    #[test]
9436    fn windows_store_touch_fixture() {
9437        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9438            std::fs::write(path, b"module ran").unwrap();
9439        }
9440    }
9441
9442    #[cfg(windows)]
9443    #[tokio::test]
9444    async fn windows_daemon_fixture() {
9445        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9446            return;
9447        };
9448        let mut spec = spec();
9449        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9450        let supervisor =
9451            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9452        let _module = supervisor.spawn(spec).unwrap();
9453    }
9454
9455    #[cfg(windows)]
9456    #[test]
9457    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9458        use std::io::Read;
9459        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9460        // An outer job models a scheduler/service host. The daemon's root and
9461        // root job must nest without requiring breakaway from this job.
9462        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9463        command
9464            .args([
9465                "--exact",
9466                "supervise::spawn_record_tests::windows_daemon_fixture",
9467                "--nocapture",
9468            ])
9469            .env(
9470                "SUBC_TEST_DAEMON_BARRIER",
9471                listener.local_addr().unwrap().to_string(),
9472            );
9473        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9474        listener.set_nonblocking(true).unwrap();
9475        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9476        let mut stream = loop {
9477            if let Ok((stream, _)) = listener.accept() {
9478                break stream;
9479            }
9480            assert!(
9481                daemon.child.try_wait().unwrap().is_none(),
9482                "daemon failed before rendezvous"
9483            );
9484            assert!(
9485                std::time::Instant::now() < deadline,
9486                "daemon did not reach CreateProcess failpoint"
9487            );
9488            std::thread::yield_now();
9489        };
9490        stream
9491            .set_read_timeout(Some(Duration::from_secs(30)))
9492            .unwrap();
9493        let mut pid = [0; 4];
9494        stream.read_exact(&mut pid).unwrap();
9495        let pid = u32::from_ne_bytes(pid);
9496        assert!(subc_jobobject::process_exists(pid));
9497        // Terminate only the daemon, not its outer job: otherwise the host job
9498        // would also kill a child whose daemon never joined its root job.
9499        daemon.child.kill().unwrap();
9500        daemon.child.wait().unwrap();
9501        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9502        if !exited {
9503            daemon.job.terminate().unwrap();
9504        }
9505        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9506    }
9507
9508    #[cfg(windows)]
9509    #[tokio::test]
9510    async fn windows_containment_failure_never_resumes() {
9511        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9512        let sentinel = home.join("store-touched");
9513        let mut spec = spec();
9514        spec.env
9515            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9516        spec.env.push((
9517            "SUBC_TEST_STORE_TOUCH".into(),
9518            sentinel.display().to_string(),
9519        ));
9520        let supervisor =
9521            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9522        let result = supervisor.spawn(spec);
9523        // A version that resumes despite assignment failure must have time to
9524        // execute its sentinel before dropping the returned module kills it.
9525        if let Ok(module) = result {
9526            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9527                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9528                    tokio::task::yield_now().await;
9529                }
9530            })
9531            .await;
9532            let _ = module.stop().await;
9533            panic!(
9534                "assignment failure resumed a module; store touched={}",
9535                sentinel.exists()
9536            );
9537        }
9538        assert!(
9539            !sentinel.exists(),
9540            "refused module executed store-touch sentinel"
9541        );
9542        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9543    }
9544
9545    #[cfg(windows)]
9546    #[tokio::test]
9547    async fn windows_nested_daemon_fixture() {
9548        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9549            return;
9550        };
9551        let mut spec = spec();
9552        spec.env.push((
9553            "SUBC_TEST_STORE_TOUCH".into(),
9554            sentinel.to_str().unwrap().into(),
9555        ));
9556        let supervisor =
9557            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9558        let module = supervisor
9559            .spawn(spec)
9560            .expect("per-module job must nest below daemon and host jobs");
9561        tokio::time::timeout(Duration::from_secs(10), async {
9562            while !PathBuf::from(&sentinel).exists() {
9563                tokio::task::yield_now().await;
9564            }
9565        })
9566        .await
9567        .unwrap();
9568        module.stop().await.unwrap();
9569    }
9570
9571    #[cfg(windows)]
9572    #[test]
9573    fn windows_module_job_nests_under_host_job() {
9574        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9575        let sentinel = home.join("nested-store-touched");
9576        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9577        command
9578            .args([
9579                "--exact",
9580                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9581                "--nocapture",
9582            ])
9583            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9584        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9585        assert!(daemon.child.wait().unwrap().success());
9586        assert!(sentinel.exists(), "nested protocol:none module never ran");
9587    }
9588}
9589
9590#[cfg(target_os = "linux")]
9591pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9592    use subc_cgroup::KillOutcome;
9593    match subc_cgroup::kill_module(placement, module_id) {
9594        KillOutcome::Killed => {}
9595        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9596            debug!(
9597                module_id,
9598                "cgroup tree kill unavailable; using direct-child kill"
9599            );
9600        }
9601        KillOutcome::IoError { path, error } => {
9602            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9603        }
9604    }
9605}
9606
9607/// Assign a freshly spawned Windows child without ever resuming on failure.
9608///
9609/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9610/// child assigned **while it is still suspended** (step 1 is
9611/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9612/// the child's roster admission is durable. Failed containment kills and reaps
9613/// the suspended child instead of running an unprotected module.
9614#[cfg(windows)]
9615fn contain_spawned_child(
9616    child: &mut Child,
9617    spec: &ModuleSpec,
9618) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9619    let assignment = || {
9620        let job = subc_jobobject::JobObject::new()?;
9621        #[cfg(test)]
9622        spawn_record_tests::assignment_failpoint(spec)?;
9623        job.assign(child)?;
9624        Ok::<_, io::Error>(job)
9625    };
9626    match assignment() {
9627        Ok(job) => Ok(Some(job)),
9628        Err(source) => {
9629            kill_and_reap_rejected_child(child);
9630            Err(SuperviseError::Spawn {
9631                program: spec.program.clone(),
9632                source: io::Error::other(format!(
9633                    "Windows module job containment failed: {source}"
9634                )),
9635                cgroup_path: None,
9636            })
9637        }
9638    }
9639}
9640
9641/// Rejection is synchronous: do not return a spawn error while the child can
9642/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9643fn kill_and_reap_rejected_child(child: &mut Child) {
9644    #[cfg(unix)]
9645    if let Some(pid) = child
9646        .id()
9647        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9648    {
9649        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9650    }
9651    let _ = child.start_kill();
9652    while matches!(child.try_wait(), Ok(None)) {
9653        std::thread::yield_now();
9654    }
9655}
9656
9657/// Resume a suspended child, killing it if it cannot be started.
9658///
9659/// A suspended process holds a pid and does nothing, so there is no useful
9660/// state to return: the caller gets an error and the spawn fails.
9661#[cfg(windows)]
9662fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9663    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9664        // Kill it here rather than leaving a suspended process for the caller
9665        // to notice; `kill_on_drop` would eventually do this, but the module
9666        // would have been reported as running in between.
9667        let _ = std::process::Command::new("taskkill.exe")
9668            .args(["/PID", &pid.to_string(), "/T", "/F"])
9669            .stdin(Stdio::null())
9670            .stdout(Stdio::null())
9671            .stderr(Stdio::null())
9672            .status();
9673        return Err(SuperviseError::Spawn {
9674            program: spec.program.clone(),
9675            source,
9676            cgroup_path: None,
9677        });
9678    }
9679    Ok(())
9680}
9681
9682#[cfg(target_os = "linux")]
9683fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9684    match placement.remove_module(module_id) {
9685        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9686        Err(error) => warn!(
9687            module_id,
9688            error = %error,
9689            "could not remove module cgroup after process exit; continuing teardown"
9690        ),
9691    }
9692}
9693
9694#[cfg(target_os = "linux")]
9695const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
9696#[cfg(target_os = "linux")]
9697const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
9698
9699#[cfg(target_os = "linux")]
9700async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9701    let path = placement.module_path(module_id).ok();
9702    if path.as_ref().is_some_and(|path| {
9703        std::fs::read_to_string(path.join("cgroup.events"))
9704            .ok()
9705            .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
9706    }) {
9707        remove_module_cgroup(placement, module_id);
9708        return;
9709    }
9710
9711    // Freeze before enumerating: a frozen task cannot exit and release its PID
9712    // for reuse by a process outside this module while SIGTERM is being sent.
9713    if terminate_reaped_cgroup(placement, module_id).await {
9714        if let Some(path) = &path {
9715            let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
9716            while cgroup_is_populated(path) && Instant::now() < deadline {
9717                sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
9718            }
9719        }
9720    }
9721
9722    // Reaping the direct child is not proof its descendants exited. Kill any
9723    // survivors; when `cgroup.kill` succeeds, wait for the kernel to report the
9724    // cgroup empty before rmdir.
9725    if matches!(
9726        subc_cgroup::kill_module(Some(placement), module_id),
9727        subc_cgroup::KillOutcome::Killed
9728    ) {
9729        if let Some(path) = &path {
9730            while cgroup_is_populated(path) {
9731                sleep(Duration::from_millis(1)).await;
9732            }
9733        }
9734    }
9735    remove_module_cgroup(placement, module_id);
9736}
9737
9738#[cfg(target_os = "linux")]
9739fn cgroup_is_populated(path: &std::path::Path) -> bool {
9740    std::fs::read_to_string(path.join("cgroup.events"))
9741        .ok()
9742        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9743}
9744
9745#[cfg(target_os = "linux")]
9746async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
9747    let frozen =
9748        match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
9749            .await
9750        {
9751            subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
9752            outcome => {
9753                tracing::debug!(
9754                    module_id,
9755                    ?outcome,
9756                    "cgroup freeze failed; skipping SIGTERM grace"
9757                );
9758                return false;
9759            }
9760        };
9761    let listed = match frozen.processes() {
9762        subc_cgroup::ProcsOutcome::Listed(pids) => {
9763            for pid in pids.into_iter().filter(|pid| *pid > 1) {
9764                if let Some(pid) = rustix::process::Pid::from_raw(pid) {
9765                    if let Err(error) =
9766                        rustix::process::kill_process(pid, rustix::process::Signal::TERM)
9767                    {
9768                        tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
9769                    }
9770                }
9771            }
9772            true
9773        }
9774        outcome => {
9775            tracing::debug!(
9776                module_id,
9777                ?outcome,
9778                "cgroup process listing failed; skipping SIGTERM grace"
9779            );
9780            false
9781        }
9782    };
9783    match frozen.thaw() {
9784        subc_cgroup::ThawOutcome::Thawed => listed,
9785        outcome => {
9786            tracing::debug!(
9787                module_id,
9788                ?outcome,
9789                "cgroup thaw failed; proceeding to cgroup.kill"
9790            );
9791            false
9792        }
9793    }
9794}
9795
9796#[cfg(target_os = "linux")]
9797fn apply_cgroup_placement(
9798    command: &mut Command,
9799    spec: &ModuleSpec,
9800    path: &std::path::Path,
9801) -> Result<(), SuperviseError> {
9802    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9803        module_id: spec.module_id.clone(),
9804        source,
9805    })
9806}
9807
9808fn capture_retention(spec: &ModuleSpec) -> Retention {
9809    let defaults = Retention::default();
9810    let value = |name: &str| {
9811        spec.env
9812            .iter()
9813            .rev()
9814            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9815    };
9816    Retention {
9817        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9818            .and_then(|value| value.parse().ok())
9819            .unwrap_or(defaults.max_file_mb),
9820        keep: value(CAPTURE_KEEP_ENV)
9821            .and_then(|value| value.parse().ok())
9822            .unwrap_or(defaults.keep),
9823        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9824            .and_then(|value| value.parse().ok())
9825            .unwrap_or(defaults.max_age_days),
9826    }
9827}
9828
9829/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9830/// module's registration to the exact process the supervisor spawned.
9831fn generate_launch_nonce() -> Result<String, SuperviseError> {
9832    let mut bytes = [0u8; 32];
9833    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9834        reason: source.to_string(),
9835    })?;
9836    let mut hex = String::with_capacity(64);
9837    for b in bytes {
9838        use std::fmt::Write;
9839        let _ = write!(hex, "{b:02x}");
9840    }
9841    Ok(hex)
9842}
9843
9844/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9845/// signal about how many leading bytes matched.
9846fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9847    if a.len() != b.len() {
9848        return false;
9849    }
9850    let mut diff = 0u8;
9851    for (x, y) in a.iter().zip(b.iter()) {
9852        diff |= x ^ y;
9853    }
9854    diff == 0
9855}
9856
9857/// The kernel's image after an acknowledged exec, shared by ordinary launches
9858/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9859/// not identities inferred from a configured pathname.
9860#[cfg(not(windows))]
9861fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9862    let process = subc_os::Process::open(pid).ok().flatten()?;
9863    #[cfg(target_os = "linux")]
9864    return sample_after_linux_exec(
9865        || {
9866            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9867            let committed = linux_exec_committed(&stat);
9868            #[cfg(test)]
9869            spawn_image_tests::report_exec_state(committed);
9870            committed
9871        },
9872        || process.observe(),
9873    );
9874    #[cfg(not(target_os = "linux"))]
9875    process.observe()
9876}
9877
9878/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9879/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9880/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9881#[cfg(target_os = "linux")]
9882fn linux_exec_committed(stat: &str) -> Option<bool> {
9883    const PF_FORKNOEXEC: u64 = 0x40;
9884    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9885    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9886}
9887
9888#[cfg(target_os = "linux")]
9889fn sample_after_linux_exec(
9890    mut exec_committed: impl FnMut() -> Option<bool>,
9891    observe: impl FnOnce() -> Option<subc_os::Observation>,
9892) -> Option<subc_os::Observation> {
9893    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9894    while !exec_committed()? {
9895        // A failed or indefinitely stalled exec must not publish the inherited
9896        // image as authority to signal a future orphan. Missing identity keeps
9897        // the existing conservative orphan-sweep behavior.
9898        if std::time::Instant::now() >= deadline {
9899            return None;
9900        }
9901        std::thread::sleep(Duration::from_micros(50));
9902    }
9903    observe()
9904}
9905
9906#[cfg(all(test, target_os = "linux"))]
9907mod spawn_image_tests {
9908    use super::*;
9909    use std::cell::{Cell, RefCell};
9910
9911    enum ExecEvent {
9912        Uncommitted,
9913        Image(Option<subc_os::Observation>),
9914    }
9915
9916    thread_local! {
9917        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9918    }
9919
9920    pub(super) fn report_exec_state(committed: Option<bool>) {
9921        if committed == Some(false) {
9922            EXEC_EVENTS.with(|events| {
9923                if let Some(events) = events.borrow_mut().take() {
9924                    let _ = events.send(ExecEvent::Uncommitted);
9925                }
9926            });
9927        }
9928    }
9929
9930    #[test]
9931    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9932        use std::io::{Read, Write};
9933        use std::os::fd::AsRawFd;
9934
9935        let descriptor = std::fs::File::open("/dev/null").unwrap();
9936        let mut command = std::process::Command::new("/bin/sleep");
9937        command.arg("30");
9938        let (mut ready, mut release) =
9939            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9940                .unwrap();
9941        let spawn = std::thread::spawn(move || command.spawn());
9942        let mut message = [0u8; 8];
9943        ready.read_exact(&mut message).unwrap();
9944        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9945        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9946        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9947        let (events, received) = std::sync::mpsc::channel();
9948        let observer = std::thread::spawn(move || {
9949            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9950            let image = observe_spawned_image(pid);
9951            let _ = events.send(ExecEvent::Image(image));
9952        });
9953        // Release only once the production observer has seen the uncommitted
9954        // exec, or has incorrectly returned the inherited image. Either way,
9955        // release and reap before asserting so a regression cannot leak a child.
9956        let first = received.recv_timeout(Duration::from_secs(5));
9957        release.write_all(&[1]).unwrap();
9958        let mut child = spawn.join().unwrap().unwrap();
9959        let after = match first {
9960            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9961                Ok(ExecEvent::Image(image)) => image,
9962                _ => None,
9963            },
9964            Ok(ExecEvent::Image(image)) => image,
9965            Err(_) => None,
9966        };
9967        observer.join().unwrap();
9968        child.kill().unwrap();
9969        child.wait().unwrap();
9970
9971        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9972        assert_eq!(
9973            inherited.unwrap().executable,
9974            subc_os::file_identity(&std::env::current_exe().unwrap())
9975        );
9976        assert_eq!(
9977            after.unwrap().executable,
9978            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9979        );
9980    }
9981
9982    #[test]
9983    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9984        let committed = Cell::new(false);
9985        let inherited = subc_os::Observation {
9986            start_time: 23,
9987            executable: Some(subc_os::FileIdentity {
9988                device: 29,
9989                inode: 31,
9990            }),
9991        };
9992        let module = subc_os::Observation {
9993            start_time: 23,
9994            executable: Some(subc_os::FileIdentity {
9995                device: 29,
9996                inode: 37,
9997            }),
9998        };
9999        let polls = Cell::new(0);
10000        let image = sample_after_linux_exec(
10001            || {
10002                polls.set(polls.get() + 1);
10003                // Hold the inherited image through the first poll, as in the
10004                // posix_spawn parent-wake window, then commit the module image.
10005                committed.set(polls.get() == 2);
10006                Some(committed.get())
10007            },
10008            || Some(if committed.get() { module } else { inherited }),
10009        );
10010        assert_eq!(image, Some(module));
10011        assert_eq!(polls.get(), 2);
10012    }
10013
10014    #[test]
10015    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10016        assert_eq!(
10017            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10018            Some(false)
10019        );
10020        assert_eq!(
10021            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10022            Some(true)
10023        );
10024        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10025        assert_eq!(linux_exec_committed("17 (a) R"), None);
10026    }
10027
10028    #[test]
10029    fn unreadable_exec_state_never_samples_an_inherited_image() {
10030        assert_eq!(
10031            sample_after_linux_exec(
10032                || None,
10033                || panic!("must not sample before exec confirmation")
10034            ),
10035            None
10036        );
10037    }
10038
10039    #[test]
10040    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10041        assert_eq!(
10042            sample_after_linux_exec(
10043                || Some(false),
10044                || panic!("must not sample after an exec timeout")
10045            ),
10046            None
10047        );
10048    }
10049}
10050
10051/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
10052/// different executable after the no-delete capture was taken.
10053#[cfg(windows)]
10054fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10055    let mut program = spec.program.clone();
10056    if program.is_absolute() || program.components().count() > 1 {
10057        // An explicit path can name a PE image with no extension. Try that
10058        // exact file first, as Command does, before falling back to .exe.
10059        match subc_os::ExecutableCapture::open(&program) {
10060            Ok(capture) => return Ok(capture),
10061            Err(error)
10062                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10063            Err(error) => return Err(error),
10064        }
10065        program.set_extension("exe");
10066        return subc_os::ExecutableCapture::open(&program);
10067    }
10068    if program.extension().is_none() {
10069        program.set_extension("exe");
10070    }
10071    let custom_path = spec
10072        .env
10073        .iter()
10074        .rev()
10075        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10076    let mut directories = Vec::new();
10077    // These are Windows' default executable search locations. A configured
10078    // PATH still overrides the inherited PATH, not the system directories.
10079    if let Some(parent) = std::env::current_exe()?.parent() {
10080        directories.push(parent.to_path_buf());
10081    }
10082    directories.push(std::env::current_dir()?);
10083    if let Some(windows) = std::env::var_os("SystemRoot") {
10084        let windows = PathBuf::from(windows);
10085        directories.push(windows.join("System32"));
10086        directories.push(windows.clone());
10087    }
10088    let path = custom_path
10089        .map(|(_, value)| std::ffi::OsString::from(value))
10090        .or_else(|| std::env::var_os("PATH"));
10091    if let Some(path) = path {
10092        directories.extend(std::env::split_paths(&path));
10093    }
10094    for directory in directories {
10095        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10096            Ok(capture) => return Ok(capture),
10097            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10098            Err(error) => return Err(error),
10099        }
10100    }
10101    Err(io::Error::new(
10102        io::ErrorKind::NotFound,
10103        "resolved executable not found",
10104    ))
10105}
10106
10107fn spawn_and_mark_running(
10108    spec: &ModuleSpec,
10109    runtime: &SupervisorRuntimeConfig,
10110    snapshot: &SharedSnapshot,
10111) -> Result<SupervisedChild, SuperviseError> {
10112    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10113    let registration_watch = watch_registration(&runtime.registry, spec);
10114    let child = spawn_child(
10115        spec,
10116        runtime.connection_file_path.as_deref(),
10117        runtime.supervisor_handle.as_ref(),
10118        &runtime.stderr_ring,
10119        runtime.capture_logs_dir.as_deref(),
10120        &runtime.child_roster,
10121        #[cfg(target_os = "linux")]
10122        runtime.cgroup_placement.as_ref(),
10123    )?;
10124    let mut child = child;
10125    arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10126    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
10127    if let Some(spawn) = activity_spawn {
10128        spawn.complete(snapshot);
10129    }
10130    Ok(child)
10131}
10132
10133struct RegistrationWatch {
10134    changes: watch::Receiver<u64>,
10135    baseline: u64,
10136}
10137
10138// Capture before spawning: a fast child's accepted HELLO can disappear from the
10139// active registry before the monitor first runs. A pre-existing registration
10140// must not satisfy a new process's deadline either.
10141fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10142    if spec.protocol != ModuleProtocol::Subc {
10143        return None;
10144    }
10145    let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10146    let baseline = *changes.borrow();
10147    Some(RegistrationWatch { changes, baseline })
10148}
10149
10150fn arm_registration_deadline(
10151    child: &mut SupervisedChild,
10152    budget: Duration,
10153    registration_watch: Option<RegistrationWatch>,
10154) {
10155    child.registration_deadline =
10156        (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10157    child.registration_watch = registration_watch;
10158}
10159
10160async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10161    #[cfg(unix)]
10162    {
10163        // Children lead their own process groups. Signal the whole group so a
10164        // helper stuck before HELLO cannot survive its supervised parent.
10165        let group = unreaped_registration_group(child)?;
10166        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10167        // Process-group safety rule: only signal a group while its leader is
10168        // our unreaped child. Even a zombie leader reserves its pid/group id;
10169        // reaping it before KILL would let that id name an unrelated group.
10170        // Sleep without waiting so TERM-ignoring helpers get the full grace.
10171        sleep(crate::child_roster::TERM_TO_KILL).await;
10172        let group = unreaped_registration_group(child)?;
10173        let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10174    }
10175    // Windows terminates the containing Job Object as well as the direct
10176    // child. Linux also kills the child's cgroup when placement is enabled.
10177    child.start_kill()?;
10178    child.wait().await?;
10179    Ok(())
10180}
10181
10182#[cfg(unix)]
10183fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10184    // The group id comes only from the pid recorded at spawn, when the child
10185    // created its own group. Never admit init's group or a negative group id.
10186    let group = i32::try_from(child.pid)
10187        .ok()
10188        .filter(|pid| *pid > 1)
10189        .and_then(rustix::process::Pid::from_raw)
10190        .ok_or_else(|| {
10191            io::Error::new(
10192                io::ErrorKind::InvalidInput,
10193                "invalid supervised process group",
10194            )
10195        })?;
10196    if child.child.id() != Some(child.pid) {
10197        return Err(io::Error::new(
10198            io::ErrorKind::InvalidInput,
10199            "refusing process-group signal after supervised child was reaped",
10200        ));
10201    }
10202    Ok(group)
10203}
10204
10205enum RegistrationWaitOutcome {
10206    Registered,
10207    Exited(ExitReport),
10208    TimedOut,
10209}
10210
10211struct ReloadRegistrationFailure {
10212    exit_report: ExitReport,
10213    reason: String,
10214}
10215
10216#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10217enum BusyGaugeObservation {
10218    Quiescent,
10219    Busy,
10220    Omitted,
10221}
10222
10223/// What one health answer said about one declared busy gauge.
10224#[derive(Debug, Clone, PartialEq, Eq)]
10225enum GaugeVerdict {
10226    /// The gauge read 0: this gauge is not what holds the drain.
10227    Zero,
10228    /// The gauge read a positive count.
10229    NonZero(u64),
10230    /// The gauge was present but not a non-negative integer (`0.0`, a string,
10231    /// a negative number). It counts as busy, because the daemon cannot tell
10232    /// it is idle. Carries the value as JSON text, cut to a short prefix.
10233    NotACount(String),
10234    /// The health answer had no metrics object, or one without this gauge.
10235    Missing,
10236    /// The probe produced no health answer to read; carries the probe error's
10237    /// evidence label (`no-answer` for a probe that timed out).
10238    ProbeFailed(&'static str),
10239}
10240
10241impl GaugeVerdict {
10242    fn reads_quiet(&self) -> bool {
10243        matches!(self, Self::Zero)
10244    }
10245}
10246
10247impl fmt::Display for GaugeVerdict {
10248    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10249        match self {
10250            Self::Zero => f.write_str("zero"),
10251            Self::NonZero(value) => write!(f, "non_zero({value})"),
10252            Self::NotACount(value) => write!(f, "not_a_count({value})"),
10253            Self::Missing => f.write_str("missing_from_health_body"),
10254            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10255        }
10256    }
10257}
10258
10259/// The longest JSON text kept for a gauge value that is not a count, so a
10260/// module reporting a large object as a gauge cannot flood the drain log.
10261const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10262
10263/// Read each declared gauge out of one health answer's metrics.
10264fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10265    let metrics = metrics.and_then(Value::as_object);
10266    gauges
10267        .iter()
10268        .map(
10269            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10270                None => GaugeVerdict::Missing,
10271                Some(value) => match value.as_u64() {
10272                    Some(0) => GaugeVerdict::Zero,
10273                    Some(count) => GaugeVerdict::NonZero(count),
10274                    None => {
10275                        let mut text = value.to_string();
10276                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10277                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
10278                            while !text.is_char_boundary(end) {
10279                                end -= 1;
10280                            }
10281                            text.truncate(end);
10282                            text.push_str("...");
10283                        }
10284                        GaugeVerdict::NotACount(text)
10285                    }
10286                },
10287            },
10288        )
10289        .collect()
10290}
10291
10292/// The drain's overall reading of one health answer: omitted when any
10293/// declared gauge is missing, quiescent only when every gauge reads 0.
10294fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10295    if verdicts
10296        .iter()
10297        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10298    {
10299        BusyGaugeObservation::Omitted
10300    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10301        BusyGaugeObservation::Quiescent
10302    } else {
10303        BusyGaugeObservation::Busy
10304    }
10305}
10306
10307/// One declared gauge's history over a drain: the latest verdict on it, and
10308/// the latest value actually read for it (which a later failed probe or
10309/// missing gauge does not erase). Times are offsets from the start of the
10310/// drain's wait.
10311#[derive(Debug, Clone, Default, PartialEq, Eq)]
10312struct GaugeTrace {
10313    latest: Option<(GaugeVerdict, Duration)>,
10314    last_value: Option<(GaugeVerdict, Duration)>,
10315}
10316
10317/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
10318/// its ceiling can say in its timeout line which declared gauge held it and
10319/// why. Without it, a gauge that read busy, a gauge the module never
10320/// reported, and a probe that never got an answer all looked the same: a
10321/// drain that ended at its ceiling with nothing in flight.
10322#[derive(Debug, Clone, Default, PartialEq, Eq)]
10323struct DrainGaugeTrace {
10324    /// Health probes the drain sent.
10325    probes: u32,
10326    /// Each declared gauge, in declaration order.
10327    gauges: Vec<(String, GaugeTrace)>,
10328}
10329
10330impl DrainGaugeTrace {
10331    fn new(gauges: &[String]) -> Self {
10332        Self {
10333            probes: 0,
10334            gauges: gauges
10335                .iter()
10336                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10337                .collect(),
10338        }
10339    }
10340
10341    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
10342    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10343        self.probes = self.probes.saturating_add(1);
10344        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10345            trace.latest = Some((verdict.clone(), at));
10346            if matches!(
10347                verdict,
10348                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10349            ) {
10350                trace.last_value = Some((verdict.clone(), at));
10351            }
10352        }
10353    }
10354}
10355
10356/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
10357/// declared gauge, separated by ` | `. A value is shown as its verdict
10358/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
10359/// and whether that counted as busy.
10360impl fmt::Display for DrainGaugeTrace {
10361    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10362        if self.gauges.is_empty() {
10363            return f.write_str("none declared");
10364        }
10365        for (index, (name, trace)) in self.gauges.iter().enumerate() {
10366            if index > 0 {
10367                f.write_str(" | ")?;
10368            }
10369            write!(f, "{name}: ")?;
10370            match &trace.latest {
10371                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10372                None => f.write_str("never probed")?,
10373            }
10374            match &trace.last_value {
10375                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10376                None => f.write_str("; no value ever read")?,
10377            }
10378        }
10379        Ok(())
10380    }
10381}
10382
10383pub(crate) fn declared_busy_gauges(
10384    registry: &Registry,
10385    module_id: &str,
10386) -> Result<Vec<String>, SuperviseError> {
10387    busy_gauges_of(
10388        registry
10389            .get_module(module_id)
10390            .map_err(SuperviseError::Registry)?,
10391    )
10392}
10393
10394/// [`declared_busy_gauges`] for the registration a connection holds, in any
10395/// slot: after cutover the incumbent is no longer the id's active
10396/// registration, and its own manifest is the one that names its gauges.
10397fn declared_busy_gauges_for_connection(
10398    registry: &Registry,
10399    connection_id: ConnectionId,
10400) -> Result<Vec<String>, SuperviseError> {
10401    busy_gauges_of(
10402        registry
10403            .get_module_by_connection(connection_id)
10404            .map_err(SuperviseError::Registry)?,
10405    )
10406}
10407
10408fn busy_gauges_of(
10409    registration: Option<crate::registry::ModuleRegistration>,
10410) -> Result<Vec<String>, SuperviseError> {
10411    let Some(registration) = registration else {
10412        return Ok(Vec::new());
10413    };
10414    let Some(self_signals) = registration.manifest.self_signals else {
10415        return Ok(Vec::new());
10416    };
10417
10418    let mut gauges = Vec::new();
10419    for declaration in self_signals {
10420        if declaration.kind != SelfSignalKind::Busy {
10421            continue;
10422        }
10423        match declaration.anchored_to {
10424            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10425                gauges.extend(declared)
10426            }
10427            _ => {
10428                // An invalid Busy anchor is fail-safe: the empty name cannot be
10429                // present in a conforming health report, so this drain stays busy.
10430                gauges.push(String::new());
10431            }
10432        }
10433    }
10434    Ok(gauges)
10435}
10436
10437/// Wait for `endpoint` to have nothing in flight and, when the module declares
10438/// busy gauges, for a health probe to report them quiet. The probe is addressed
10439/// by `scope`: a swap's superseded incumbent must be asked about its own
10440/// gauges, and by module id the probe would reach the promoted candidate.
10441/// While the gauges read busy, the probe repeats every
10442/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
10443/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
10444/// timeout line.
10445#[allow(clippy::too_many_arguments)]
10446async fn wait_for_forwarding_quiescence(
10447    forwarding: &ForwardingTable,
10448    module_id: &str,
10449    runtime: &SupervisorRuntimeConfig,
10450    endpoint: crate::ModuleEndpointId,
10451    deadline: Instant,
10452    busy_gauges: &[String],
10453    scope: DrainScope,
10454    trace: &mut DrainGaugeTrace,
10455) -> Result<bool, SuperviseError> {
10456    let wait_started = Instant::now();
10457    let mut gauges_quiescent = busy_gauges.is_empty();
10458    let mut next_probe_at = wait_started;
10459    let mut omission_reported = false;
10460
10461    loop {
10462        let now = Instant::now();
10463        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10464            let report = match scope {
10465                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10466                DrainScope::Endpoint(endpoint) => {
10467                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10468                }
10469            };
10470            let verdicts = match &report {
10471                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10472                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10473            };
10474            trace.record(&verdicts, wait_started.elapsed());
10475            gauges_quiescent = match report {
10476                Ok(_) => match busy_gauge_observation(&verdicts) {
10477                    BusyGaugeObservation::Quiescent => true,
10478                    BusyGaugeObservation::Busy => false,
10479                    BusyGaugeObservation::Omitted => {
10480                        // Once per drain: a module that never reports a gauge
10481                        // it declared would otherwise say so on every probe.
10482                        if !omission_reported {
10483                            forwarding
10484                                .counters()
10485                                .increment_drains_with_undeclared_gauge();
10486                            let missing = busy_gauges
10487                                .iter()
10488                                .zip(&verdicts)
10489                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10490                                .map(|(gauge, _)| gauge.as_str())
10491                                .collect::<Vec<_>>()
10492                                .join(",");
10493                            warn!(
10494                                module_id,
10495                                missing_gauges = %missing,
10496                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10497                            );
10498                            omission_reported = true;
10499                        }
10500                        false
10501                    }
10502                },
10503                Err(err) => {
10504                    warn!(
10505                        module_id,
10506                        error = %err,
10507                        "drain health.check did not produce declared busy gauges; treating module as busy"
10508                    );
10509                    false
10510                }
10511            };
10512            // Counted from the probe's completion: the probe above was awaited,
10513            // so no second probe can start while one is still outstanding.
10514            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10515        }
10516
10517        let in_flight = forwarding
10518            .endpoint_in_flight_count(endpoint)
10519            .map_err(SuperviseError::Forwarding)?;
10520        if in_flight == 0 && gauges_quiescent {
10521            return Ok(true);
10522        }
10523
10524        let now = Instant::now();
10525        if now >= deadline {
10526            return Ok(false);
10527        }
10528        let mut wait = deadline
10529            .saturating_duration_since(now)
10530            .min(REGISTRY_RELEASE_POLL);
10531        if !busy_gauges.is_empty() {
10532            wait = wait.min(next_probe_at.saturating_duration_since(now));
10533        }
10534        sleep(wait).await;
10535    }
10536}
10537
10538/// The wait between drain busy-gauge probes: the larger of
10539/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10540/// drain's own in-flight poll interval.
10541fn drain_busy_reprobe_interval() -> Duration {
10542    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10543}
10544
10545/// Logs why a drain gave up: the drain waited its full timeout without the
10546/// module going quiet, and teardown is being forced.
10547///
10548/// The line names two kinds of evidence, because a bare "timed out" hides
10549/// causes that need different fixes:
10550/// - `holdouts`: the requests and routes still open when the timeout hit,
10551///   and on which connections. One stuck call is a module bug; a held stream
10552///   on every session is a module that should end its streams when it
10553///   receives `module.draining`.
10554/// - `gauges`: for each busy gauge the module declares, its latest reading
10555///   and how many probes ran. This separates a module that kept reporting
10556///   itself busy, one whose health answer never contained the gauge, and one
10557///   that never answered the probes, which otherwise all look like a timeout
10558///   with nothing in flight.
10559fn warn_drain_timed_out(
10560    module_id: &str,
10561    drain_timeout: Duration,
10562    reason: RouteCloseReason,
10563    holdouts: &crate::forwarding::DrainHoldouts,
10564    gauges: &DrainGaugeTrace,
10565) {
10566    warn!(
10567        module_id = %module_id,
10568        waited = ?drain_timeout,
10569        ?reason,
10570        held_requests = holdouts.requests,
10571        held_routes = holdouts.routes,
10572        total_routes = holdouts.total_routes,
10573        top_connections = ?holdouts.top_connections,
10574        // `module_channel:corr`, so the module can find each held request
10575        // in its own log; capped, so `held_requests` is the full count.
10576        held = %holdouts
10577            .held
10578            .iter()
10579            .map(|(channel, corr)| format!("{channel}:{corr}"))
10580            .collect::<Vec<_>>()
10581            .join(","),
10582        gauge_probes = gauges.probes,
10583        busy_gauges = %gauges,
10584        "route drain timed out before request quiescence; forcing teardown"
10585    );
10586}
10587
10588/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10589///
10590/// `Ok` is always honest and passed straight through -- the wait actually measured
10591/// in-flight state. `Err` means the wait produced no measurement at all (the
10592/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10593/// constant: the drain did not complete. Never recomputed from route state, never a
10594/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10595fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10596    match wait_result {
10597        Ok(drained) => *drained,
10598        Err(_) => false,
10599    }
10600}
10601
10602fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10603    for released in released_routes {
10604        let frame = match Frame::build_with_version(
10605            released.negotiated_ver,
10606            FrameType::Goodbye,
10607            control_flags(),
10608            released.channel,
10609            released.epoch,
10610            0,
10611            Vec::new(),
10612        ) {
10613            Ok(frame) => frame,
10614            Err(err) => {
10615                warn!(
10616                    route_channel = released.channel,
10617                    error = %err,
10618                    "failed to build supervisor drain route GOODBYE frame"
10619                );
10620                continue;
10621            }
10622        };
10623        if !released.close_on_delivery_failure() {
10624            crate::forwarding::send_module_route_goodbye(
10625                &forwarding.counters(),
10626                &released.sink,
10627                frame,
10628                released.module_id.as_deref(),
10629                "supervisor drain",
10630            );
10631            continue;
10632        }
10633        if let Err(err) = released.sink.try_send(frame) {
10634            warn!(
10635                target_connection_id = released.connection_id.get(),
10636                route_channel = released.channel,
10637                error = %err,
10638                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10639            );
10640            let _ = forwarding.escalate_client_delivery_failure(
10641                released.connection_id,
10642                released.channel,
10643                released.epoch,
10644                CloseReason::new(
10645                    "route_goodbye_delivery_failed",
10646                    format!(
10647                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10648                        released.channel
10649                    ),
10650                ),
10651                crate::forwarding::UndeliveredFrame {
10652                    module_id: released.module_id.as_deref(),
10653                    sink: &released.sink,
10654                },
10655            );
10656        }
10657    }
10658}
10659
10660async fn send_module_draining(
10661    module_id: &str,
10662    reason: RouteCloseReason,
10663    deadline_ms: u64,
10664    target: &ModuleDrainTarget,
10665    _wait_for_flush: bool,
10666) -> bool {
10667    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10668        reason,
10669        deadline_ms,
10670    }) {
10671        Ok(body) => body,
10672        Err(err) => {
10673            warn!(
10674                module_id,
10675                error = %err,
10676                "failed to encode module draining command"
10677            );
10678            return false;
10679        }
10680    };
10681    let frame = match Frame::build_with_version(
10682        target.negotiated_ver,
10683        FrameType::Push,
10684        control_flags(),
10685        0,
10686        0,
10687        0,
10688        body,
10689    ) {
10690        Ok(frame) => frame,
10691        Err(err) => {
10692            warn!(
10693                module_id,
10694                error = %err,
10695                "failed to build module draining command frame"
10696            );
10697            return false;
10698        }
10699    };
10700    #[cfg(unix)]
10701    if _wait_for_flush {
10702        return target.sink.send_flushed(frame).await.is_ok();
10703    }
10704    if let Err(err) = target.sink.try_send(frame) {
10705        warn!(
10706            module_id,
10707            target_connection_id = target.endpoint.connection_id.get(),
10708            error = %err,
10709            "module draining command was not delivered to peer"
10710        );
10711        return false;
10712    }
10713    true
10714}
10715
10716/// The channel-0 GOODBYE that tells a module its stop is planned.
10717fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10718    match Frame::build_with_version(
10719        negotiated_ver,
10720        FrameType::Goodbye,
10721        control_flags(),
10722        0,
10723        0,
10724        0,
10725        Vec::new(),
10726    ) {
10727        Ok(frame) => Some(frame),
10728        Err(err) => {
10729            warn!(
10730                module_id,
10731                error = %err,
10732                "failed to build module GOODBYE frame"
10733            );
10734            None
10735        }
10736    }
10737}
10738
10739/// Send every registered module connection its module GOODBYE at daemon
10740/// shutdown, then request that connection's close.
10741///
10742/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10743/// before EOF, so the GOODBYE must reach the socket before the close. A close
10744/// request does not wait for the connection's queued frames: its writer gets a
10745/// bounded grace after the close, is aborted if it overruns it, and the daemon
10746/// process may exit before that grace ends. So with `wait_for_flush`, each
10747/// connection is closed only after its writer has acknowledged writing the
10748/// GOODBYE, or once a short shared budget runs out, so one module that is not
10749/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10750/// are only queued, for a shutdown the operator has told to stop waiting.
10751/// A connection that is already gone is skipped.
10752#[cfg(unix)]
10753async fn send_module_goodbyes_for_daemon_shutdown(
10754    forwarding: &Arc<ForwardingTable>,
10755    reason: &CloseReason,
10756    wait_for_flush: bool,
10757) {
10758    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10759    let targets = match forwarding.module_connections() {
10760        Ok(targets) => targets,
10761        Err(err) => {
10762            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10763            return;
10764        }
10765    };
10766    let deadline = Instant::now() + GOODBYE_BUDGET;
10767    let mut sends = tokio::task::JoinSet::new();
10768    for target in targets {
10769        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10770            continue;
10771        };
10772        if !wait_for_flush {
10773            if let Err(err) = target.sink.try_send(frame) {
10774                debug!(
10775                    module_id = %target.module_id,
10776                    error = %err,
10777                    "shutdown module GOODBYE was not queued"
10778                );
10779            }
10780            continue;
10781        }
10782        let forwarding = Arc::clone(forwarding);
10783        let reason = reason.clone();
10784        sends.spawn(async move {
10785            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10786                Ok(Ok(())) => {}
10787                Ok(Err(err)) => debug!(
10788                    module_id = %target.module_id,
10789                    error = %err,
10790                    "module connection closed before its shutdown GOODBYE was written"
10791                ),
10792                Err(_) => warn!(
10793                    module_id = %target.module_id,
10794                    budget = ?GOODBYE_BUDGET,
10795                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10796                ),
10797            }
10798            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10799        });
10800    }
10801    // Every task ends by the shared deadline, so this wait is bounded too.
10802    while sends.join_next().await.is_some() {}
10803}
10804
10805fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10806    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10807        return;
10808    };
10809    if let Err(err) = target.sink.try_send(frame) {
10810        warn!(
10811            module_id,
10812            target_connection_id = target.endpoint.connection_id.get(),
10813            error = %err,
10814            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10815        );
10816        forwarding.request_connection_close(
10817            target.endpoint.connection_id,
10818            CloseReason::new(
10819                "module_goodbye_delivery_failed",
10820                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10821            ),
10822        );
10823    }
10824}
10825
10826#[derive(Clone, Copy)]
10827struct ForwardingDrainContext<'a> {
10828    spec: &'a ModuleSpec,
10829    runtime: &'a SupervisorRuntimeConfig,
10830    registry: &'a Registry,
10831    scope: DrainScope,
10832}
10833
10834/// Which process a forwarding drain addresses.
10835#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10836enum DrainScope {
10837    /// Whatever endpoint is active for the module id: every plain stop,
10838    /// restart and reload. Also moves the module's state to `Draining`.
10839    Active,
10840    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10841    /// module id would resolve to the promoted candidate and leave neither
10842    /// process routable. The module's state is left alone, since the promoted
10843    /// candidate is what it describes and that process is running.
10844    Endpoint(crate::ModuleEndpointId),
10845}
10846
10847/// Whether a child being drained has already been asked to stop by the time
10848/// its drain wait starts.
10849///
10850/// The drain wait is the same budget whatever this says. What it decides is
10851/// whether the supervisor must ask by signal before that wait begins: a child
10852/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10853/// healthy or not.
10854#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10855enum StopNotice {
10856    /// The module was sent `module.draining` and a module GOODBYE over its own
10857    /// registered connection, and stops itself.
10858    SentOverConnection,
10859    /// The forwarding drain found no registered connection for the module: a
10860    /// subc child spawned moments ago that has not sent HELLO yet, or a
10861    /// `protocol: "none"` child, which never registers.
10862    NoConnection,
10863    /// This path sends nothing over the module's connection: the supervisor has
10864    /// no forwarding table, or the caller stops the child without a forwarding
10865    /// drain.
10866    NotSent,
10867}
10868
10869async fn begin_forwarding_drain(
10870    spec: &ModuleSpec,
10871    runtime: &SupervisorRuntimeConfig,
10872    registry: &Registry,
10873    snapshot: &SharedSnapshot,
10874    enabled: Option<bool>,
10875    reason: RouteCloseReason,
10876) -> Result<StopNotice, SuperviseError> {
10877    let Some(forwarding) = runtime.forwarding.as_ref() else {
10878        return Err(SuperviseError::ReloadUnavailable {
10879            module_id: spec.module_id.clone(),
10880            reason: "supervisor was not configured with a forwarding table".to_string(),
10881        });
10882    };
10883
10884    begin_forwarding_drain_with(
10885        forwarding,
10886        ForwardingDrainContext {
10887            spec,
10888            runtime,
10889            registry,
10890            scope: DrainScope::Active,
10891        },
10892        snapshot,
10893        enabled,
10894        reason,
10895        runtime.drain_timeout,
10896    )
10897    .await
10898}
10899
10900async fn begin_forwarding_drain_if_configured(
10901    spec: &ModuleSpec,
10902    runtime: &SupervisorRuntimeConfig,
10903    registry: &Registry,
10904    snapshot: &SharedSnapshot,
10905    enabled: Option<bool>,
10906    reason: RouteCloseReason,
10907) -> Result<StopNotice, SuperviseError> {
10908    begin_forwarding_drain_with_timeout(
10909        spec,
10910        runtime,
10911        registry,
10912        snapshot,
10913        enabled,
10914        reason,
10915        runtime.drain_timeout,
10916    )
10917    .await
10918}
10919
10920/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10921/// budget, for paths where the operator overrides the module's configured one
10922/// (`supervisor.restart{drain_timeout_ms}`).
10923async fn begin_forwarding_drain_with_timeout(
10924    spec: &ModuleSpec,
10925    runtime: &SupervisorRuntimeConfig,
10926    registry: &Registry,
10927    snapshot: &SharedSnapshot,
10928    enabled: Option<bool>,
10929    reason: RouteCloseReason,
10930    drain_timeout: Duration,
10931) -> Result<StopNotice, SuperviseError> {
10932    let Some(forwarding) = runtime.forwarding.as_ref() else {
10933        return Ok(StopNotice::NotSent);
10934    };
10935
10936    begin_forwarding_drain_with(
10937        forwarding,
10938        ForwardingDrainContext {
10939            spec,
10940            runtime,
10941            registry,
10942            scope: DrainScope::Active,
10943        },
10944        snapshot,
10945        enabled,
10946        reason,
10947        drain_timeout,
10948    )
10949    .await
10950}
10951
10952async fn begin_forwarding_drain_with(
10953    forwarding: &ForwardingTable,
10954    context: ForwardingDrainContext<'_>,
10955    snapshot: &SharedSnapshot,
10956    enabled: Option<bool>,
10957    reason: RouteCloseReason,
10958    drain_timeout: Duration,
10959) -> Result<StopNotice, SuperviseError> {
10960    let ForwardingDrainContext {
10961        spec,
10962        runtime,
10963        registry,
10964        scope,
10965    } = context;
10966    debug_assert_ne!(reason, RouteCloseReason::Crash);
10967    let terminal = matches!(reason, RouteCloseReason::Disable);
10968    let drain_started_at = Instant::now();
10969    let drain_deadline = drain_started_at + drain_timeout;
10970    let deadline_ms =
10971        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10972    let busy_gauges = match scope {
10973        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10974        DrainScope::Endpoint(endpoint) => {
10975            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10976        }
10977    };
10978
10979    // Admission gate first: route.open/commit and route REQUEST admission are closed
10980    // before the first quiescence check, so the outstanding count can only fall.
10981    let gate_started = Instant::now();
10982    let drain_target = match scope {
10983        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10984        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10985    }
10986    .map_err(SuperviseError::Forwarding)?;
10987    // The instant admission closed, and how long taking the forwarding write
10988    // lock to close it took. The timeout line reports only the quiescence
10989    // wait, so without this a drain that started late looked like one that
10990    // started on time.
10991    info!(
10992        module_id = %spec.module_id,
10993        ?reason,
10994        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10995        connected = drain_target.is_some(),
10996        "module drain began; route admission closed"
10997    );
10998    if scope == DrainScope::Active {
10999        update_snapshot(snapshot, Some(&spec.module_id), |state| {
11000            state.state = ModuleState::Draining;
11001            state.draining_to_replace =
11002                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11003            if let Some(enabled) = enabled {
11004                state.enabled = enabled;
11005            }
11006        })?;
11007    }
11008
11009    let Some(target) = drain_target.as_ref() else {
11010        // Nothing was sent: the module has no registered connection to carry
11011        // `module.draining` or a GOODBYE. The caller must not assume the child
11012        // was asked to stop.
11013        return Ok(StopNotice::NoConnection);
11014    };
11015    {
11016        send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11017        #[cfg(feature = "test-support")]
11018        forwarding.wait_before_route_closing_for_test().await;
11019        let routes = forwarding
11020            .endpoint_routes(target.endpoint)
11021            .map_err(SuperviseError::Forwarding)?;
11022        let routes_notified = routes.len();
11023        crate::control::send_route_control_pushes(
11024            forwarding,
11025            routes.clone(),
11026            ClientControlPush::RouteClosing {
11027                module_id: spec.module_id.clone(),
11028                channels: Vec::new(),
11029                reason,
11030            },
11031        );
11032        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11033
11034        // `route.closing` was just sent above: from here on every return path,
11035        // including an early one, MUST send `route.closed` before propagating
11036        // anything else. A client holds `closing` as a promise that a verdict is
11037        // coming; leaving early without `closed` strands it waiting forever, since
11038        // `closing` carries no timeout of its own.
11039        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11040        let wait_result = wait_for_forwarding_quiescence(
11041            forwarding,
11042            &spec.module_id,
11043            runtime,
11044            target.endpoint,
11045            drain_deadline,
11046            &busy_gauges,
11047            scope,
11048            &mut gauge_trace,
11049        )
11050        .await;
11051        let drained = drained_after_quiescence_wait(&wait_result);
11052        if let Err(err) = &wait_result {
11053            error!(
11054                module_id = %spec.module_id,
11055                ?reason,
11056                error = %err,
11057                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11058            );
11059        } else if !drained {
11060            // Read before teardown releases the routes.
11061            let holdouts = forwarding
11062                .endpoint_drain_holdouts(target.endpoint)
11063                .unwrap_or_default();
11064            warn_drain_timed_out(
11065                &spec.module_id,
11066                drain_timeout,
11067                reason,
11068                &holdouts,
11069                &gauge_trace,
11070            );
11071        }
11072        crate::control::send_route_control_pushes(
11073            forwarding,
11074            routes,
11075            ClientControlPush::RouteClosed {
11076                module_id: spec.module_id.clone(),
11077                channels: Vec::new(),
11078                reason,
11079                drained,
11080                abandoned: target.abandoned_bindings.len() as u32,
11081                excluded_subscriptions: target.excluded_subscriptions,
11082                terminal: Some(terminal),
11083            },
11084        );
11085        wait_result?;
11086
11087        // `route.closed` has now been sent unconditionally above. From here the
11088        // remaining steps are cleanup (route + module GOODBYE) rather than a
11089        // promise the client is waiting on, but a lock-poisoned
11090        // `release_module_endpoint_routes` would otherwise skip the module
11091        // GOODBYE silently too -- send it before propagating the error.
11092        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11093            Ok(routes) => routes,
11094            Err(err) => {
11095                warn!(
11096                    module_id = %spec.module_id,
11097                    ?reason,
11098                    error = %err,
11099                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11100                );
11101                send_module_goodbye(&spec.module_id, forwarding, target);
11102                return Err(SuperviseError::Forwarding(err));
11103            }
11104        };
11105        let route_goodbye_count = released_routes.len();
11106        send_route_goodbyes(forwarding, released_routes);
11107        send_module_goodbye(&spec.module_id, forwarding, target);
11108
11109        // The drain's happy path was previously silent: every emission above is
11110        // best-effort with only its failure arm logged, so "were consumers told"
11111        // was unprovable from the daemon log (surfaced by a 30-minute consumer
11112        // hang where the open question was exactly whether teardown notice went
11113        // out). One summary line makes that class decidable in one grep.
11114        info!(
11115            module_id = %spec.module_id,
11116            ?reason,
11117            routes_notified,
11118            route_goodbyes = route_goodbye_count,
11119            abandoned_reservations = target.abandoned_bindings.len(),
11120            excluded_subscriptions = target.excluded_subscriptions,
11121            drained,
11122            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11123        );
11124    }
11125
11126    Ok(StopNotice::SentOverConnection)
11127}
11128
11129/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
11130/// the only slot a plain (non-swap) spawn can register into.
11131async fn wait_for_registration_after_reload(
11132    registry: &Registry,
11133    module_id: &str,
11134    snapshot: &SharedSnapshot,
11135    child: &mut SupervisedChild,
11136    wait: Duration,
11137) -> Result<RegistrationWaitOutcome, SuperviseError> {
11138    wait_for_slot_registration(
11139        registry,
11140        crate::registry::RegistrationSlot::Active(module_id),
11141        module_id,
11142        snapshot,
11143        child,
11144        wait,
11145    )
11146    .await
11147}
11148
11149/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
11150///
11151/// Keyed on the slot rather than the bare module id because during a swap the
11152/// id's active slot is already held by the incumbent: an id-keyed wait would
11153/// report the incumbent's registration as the candidate's and a candidate that
11154/// never registers would look registered. A swap candidate waits on
11155/// `crate::registry::RegistrationSlot::Candidate`.
11156async fn wait_for_slot_registration(
11157    registry: &Registry,
11158    slot: crate::registry::RegistrationSlot<'_>,
11159    module_id: &str,
11160    snapshot: &SharedSnapshot,
11161    child: &mut SupervisedChild,
11162    wait: Duration,
11163) -> Result<RegistrationWaitOutcome, SuperviseError> {
11164    let deadline = Instant::now() + wait;
11165    loop {
11166        if registry
11167            .registration(slot)
11168            .map_err(SuperviseError::Registry)?
11169            .is_some()
11170        {
11171            return Ok(RegistrationWaitOutcome::Registered);
11172        }
11173
11174        let now = Instant::now();
11175        if now >= deadline {
11176            return Ok(RegistrationWaitOutcome::TimedOut);
11177        }
11178        let remaining = deadline.saturating_duration_since(now);
11179        let poll = remaining.min(REGISTRY_RELEASE_POLL);
11180
11181        tokio::select! {
11182            wait_result = child.wait() => {
11183                let status = wait_result.map_err(|source| SuperviseError::Wait {
11184                    module_id: module_id.to_string(),
11185                    source,
11186                })?;
11187                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11188                    snapshot,
11189                    child,
11190                    &status,
11191                )));
11192            }
11193            _ = sleep(poll) => {}
11194        }
11195    }
11196}
11197
11198fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11199    // A process stopped for missing its first HELLO did not provide service,
11200    // even if its TERM handler exits zero. Reload also uses this classification
11201    // when a replacement exits before registering.
11202    if exit_report.kind != ExitKind::DeliberateSeverance {
11203        exit_report.kind = ExitKind::Crash;
11204    }
11205    exit_report
11206}
11207
11208async fn handle_reload_child_registration_failure(
11209    spec: &ModuleSpec,
11210    runtime: &SupervisorRuntimeConfig,
11211    registry: &Registry,
11212    process_liveness: &SupervisorProcessLiveness,
11213    snapshot: &SharedSnapshot,
11214    _child: &mut Option<SupervisedChild>,
11215    failure: ReloadRegistrationFailure,
11216) -> Result<(), SuperviseError> {
11217    let ReloadRegistrationFailure {
11218        exit_report,
11219        reason,
11220    } = failure;
11221    match on_child_exit(
11222        spec,
11223        runtime.restart_policy,
11224        registry,
11225        snapshot,
11226        &runtime.terminal_ring,
11227        &runtime.spawn_events,
11228        &runtime.child_roster,
11229        exit_report,
11230        runtime.recovery_is_daemon,
11231    )
11232    .await
11233    {
11234        NextAction::Stop {
11235            registration_released,
11236        } => {
11237            if registration_released {
11238                process_liveness.untrack_if_current(&spec.module_id, snapshot);
11239            }
11240        }
11241        NextAction::Restart { schedule } => {
11242            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11243                schedule.delay
11244            });
11245            if let Some(schedule) = schedule {
11246                log_crash_respawn(&spec.module_id, schedule);
11247            }
11248            schedule_respawn(
11249                runtime,
11250                snapshot,
11251                &spec.module_id,
11252                delay,
11253                RespawnKind::Spawn,
11254                true,
11255            )?;
11256        }
11257    }
11258    Err(SuperviseError::ReloadFailed {
11259        module_id: spec.module_id.clone(),
11260        reason,
11261    })
11262}
11263
11264async fn handle_reload_spawn_failure(
11265    spec: &ModuleSpec,
11266    runtime: &SupervisorRuntimeConfig,
11267    process_liveness: &SupervisorProcessLiveness,
11268    snapshot: &SharedSnapshot,
11269    _child: &mut Option<SupervisedChild>,
11270    reason: String,
11271) -> Result<(), SuperviseError> {
11272    let now = Instant::now();
11273    let mut schedule = None;
11274    update_snapshot(snapshot, Some(&spec.module_id), |state| {
11275        clear_current_process_facts(state);
11276        if state.enabled {
11277            schedule = state.next_crash_restart(&runtime.restart_policy, now);
11278            state.state = if schedule.is_some() {
11279                ModuleState::Restarting
11280            } else {
11281                if runtime
11282                    .failure_context(runtime.recovery_is_daemon)
11283                    .may_retry()
11284                {
11285                    state.failed_retry.schedule(&spec.module_id);
11286                }
11287                ModuleState::Failed
11288            };
11289        } else {
11290            state.state = ModuleState::Disabled;
11291        }
11292    })?;
11293    if let Some(schedule) = schedule {
11294        schedule_respawn(
11295            runtime,
11296            snapshot,
11297            &spec.module_id,
11298            schedule.delay,
11299            RespawnKind::Spawn,
11300            true,
11301        )?;
11302    } else {
11303        process_liveness.untrack_if_current(&spec.module_id, snapshot);
11304    }
11305    Err(SuperviseError::ReloadFailed {
11306        module_id: spec.module_id.clone(),
11307        reason,
11308    })
11309}
11310
11311fn control_flags() -> Flags {
11312    Flags::new(false, Priority::Passive, false)
11313}
11314
11315#[allow(clippy::too_many_arguments)]
11316async fn drain_optional_child(
11317    module_id: &str,
11318    protocol: ModuleProtocol,
11319    stop_notice: StopNotice,
11320    registry: &Registry,
11321    forwarding: Option<&ForwardingTable>,
11322    snapshot: &SharedSnapshot,
11323    terminal_ring: &Arc<Mutex<TerminalRing>>,
11324    spawn_events: &SpawnEventFeed,
11325    child: &mut Option<SupervisedChild>,
11326    drain_timeout: Duration,
11327    final_state: ModuleState,
11328    enabled: Option<bool>,
11329    failure: &FailureContext,
11330) -> Result<(), SuperviseError> {
11331    if let Some(child) = child.take() {
11332        drain_child_to_state(
11333            module_id,
11334            protocol,
11335            stop_notice,
11336            registry,
11337            forwarding,
11338            snapshot,
11339            terminal_ring,
11340            spawn_events,
11341            child,
11342            drain_timeout,
11343            final_state,
11344            enabled,
11345            failure,
11346        )
11347        .await
11348    } else {
11349        update_snapshot(snapshot, Some(module_id), |state| {
11350            state.state = final_state;
11351            if final_state == ModuleState::Failed {
11352                state.failed_retry.failed_at = Some(Instant::now());
11353            }
11354            if let Some(enabled) = enabled {
11355                state.enabled = enabled;
11356            }
11357            clear_current_process_facts(state);
11358        })?;
11359        release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11360    }
11361}
11362
11363#[allow(clippy::too_many_arguments)]
11364async fn drain_child_to_state(
11365    module_id: &str,
11366    _protocol: ModuleProtocol,
11367    stop_notice: StopNotice,
11368    registry: &Registry,
11369    forwarding: Option<&ForwardingTable>,
11370    snapshot: &SharedSnapshot,
11371    terminal_ring: &Arc<Mutex<TerminalRing>>,
11372    spawn_events: &SpawnEventFeed,
11373    mut child: SupervisedChild,
11374    drain_timeout: Duration,
11375    final_state: ModuleState,
11376    enabled: Option<bool>,
11377    failure: &FailureContext,
11378) -> Result<(), SuperviseError> {
11379    let protocol = child.protocol;
11380    update_snapshot(snapshot, Some(module_id), |state| {
11381        state.state = ModuleState::Draining;
11382        state.draining_to_replace = final_state == ModuleState::Restarting;
11383        if let Some(enabled) = enabled {
11384            state.enabled = enabled;
11385        }
11386    })?;
11387
11388    // The wait below is the same budget in every case; what differs is
11389    // whether anything has ASKED the child to stop before it starts. Only a
11390    // forwarding drain that reached the module's registered connection has
11391    // (`module.draining`, then a module GOODBYE). Every other child was told
11392    // nothing: a `protocol: "none"` module, which never registers; a subc
11393    // module spawned moments ago that has not sent HELLO yet; or a stop that
11394    // runs no forwarding drain. Without a signal the budget is only a delay
11395    // in front of SIGKILL -- and the not-yet-registered child is the worst
11396    // case, because it registers into a module that is already draining,
11397    // is never told, and is killed while healthy.
11398    if stop_notice != StopNotice::SentOverConnection {
11399        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11400            info!(
11401                module_id,
11402                pid = child.pid,
11403                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11404                "module has no connection yet; requesting stop by signal"
11405            );
11406        }
11407        request_graceful_stop(module_id, &child);
11408    }
11409
11410    let exit_report = match timeout(drain_timeout, child.wait()).await {
11411        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11412        Ok(Err(source)) => {
11413            fail_after_wait_error(
11414                module_id,
11415                snapshot,
11416                terminal_ring,
11417                spawn_events,
11418                format!("failed to wait during drain: {source}"),
11419                failure,
11420            );
11421            return Err(SuperviseError::Wait {
11422                module_id: module_id.to_string(),
11423                source,
11424            });
11425        }
11426        Err(_) => {
11427            // Mirror the sibling arm above: state is already `Draining`, and an
11428            // error propagated from here would strand it there -- a state
11429            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
11430            // `Failed | Stopped`), leaving an operator Restart as the only exit.
11431            // `Failed` before `?` keeps the module operator-visible and
11432            // revivable. Trigger is an ESRCH race (process exits between the
11433            // drain timeout firing and the kill) or a post-kill wait failure
11434            // (issue #34).
11435            //
11436            // Logged because the kill is otherwise visible only as signal 9 in
11437            // the terminal ring, and the budget it follows can be long enough
11438            // that consumers see a stretch of refusals with no stated cause.
11439            warn!(
11440                module_id,
11441                pid = child.pid,
11442                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11443                reason = ?final_state,
11444                ?stop_notice,
11445                "drain budget expired before the module exited; killing it"
11446            );
11447            child.start_kill().map_err(|source| {
11448                fail_after_wait_error(
11449                    module_id,
11450                    snapshot,
11451                    terminal_ring,
11452                    spawn_events,
11453                    format!("failed to kill during drain: {source}"),
11454                    failure,
11455                );
11456                SuperviseError::Kill {
11457                    module_id: module_id.to_string(),
11458                    source,
11459                }
11460            })?;
11461            let status = child.wait().await.map_err(|source| {
11462                fail_after_wait_error(
11463                    module_id,
11464                    snapshot,
11465                    terminal_ring,
11466                    spawn_events,
11467                    format!("failed to wait after drain kill: {source}"),
11468                    failure,
11469                );
11470                SuperviseError::Wait {
11471                    module_id: module_id.to_string(),
11472                    source,
11473                }
11474            })?;
11475            classify_reaped_child_exit(snapshot, &child, &status)
11476        }
11477    };
11478
11479    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11480    publish_after_terminal_record(
11481        module_id,
11482        || {
11483            record_terminal_with_detail(
11484                module_id,
11485                terminal_ring,
11486                spawn_events,
11487                &exit_report,
11488                terminal_disposition(final_state),
11489                detail,
11490            )
11491        },
11492        || {
11493            update_snapshot(snapshot, Some(module_id), |state| {
11494                state.state = final_state;
11495                if final_state == ModuleState::Failed {
11496                    state.failed_retry.failed_at = Some(Instant::now());
11497                }
11498                if let Some(enabled) = enabled {
11499                    state.enabled = enabled;
11500                }
11501                clear_current_process_facts(state);
11502                state.last_exit = Some(exit_report.clone());
11503                if exit_report.kind == ExitKind::DeliberateSeverance {
11504                    state.lifetime_restarts += 1;
11505                }
11506            })
11507        },
11508    )?;
11509    child.drain_stderr(module_id).await;
11510
11511    release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11512}
11513
11514/// Ask a child that nothing else has asked to stop, by signal.
11515///
11516/// A registered subc module is asked over its own connection: the drain sends
11517/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
11518/// module GOODBYE, and the module stops itself. A module that speaks no subc
11519/// wire receives none of that, and neither does a subc module that has not
11520/// registered yet, so for them the drain budget would be pure delay in front of
11521/// a SIGKILL -- and for a process with a store to flush (JetStream is the
11522/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
11523/// into a recovery on the next start.
11524///
11525/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
11526/// rule rather than an optimisation: that module's graceful stop is already
11527/// running by the time its child is drained, and a signal would race it.
11528///
11529/// Best-effort by construction. A child that has already exited is the ordinary
11530/// case rather than an error (the kill lands on a reaped or exiting pid), so a
11531/// failure is logged at debug and the wait-then-kill below still decides the
11532/// outcome.
11533#[cfg(unix)]
11534fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11535    let Some(pid) = child
11536        .id()
11537        .and_then(|pid| i32::try_from(pid).ok())
11538        .and_then(rustix::process::Pid::from_raw)
11539    else {
11540        debug!(
11541            module_id,
11542            "no pid to signal for teardown; falling through to the drain wait"
11543        );
11544        return;
11545    };
11546    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11547        Ok(()) => debug!(
11548            module_id,
11549            "sent SIGTERM to a module nothing else asked to stop"
11550        ),
11551        Err(err) => debug!(
11552            module_id,
11553            error = %err,
11554            "SIGTERM to module failed; the drain wait and kill still apply"
11555        ),
11556    }
11557}
11558
11559/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11560/// Windows does offer need cooperation this supervisor cannot assume: a console
11561/// control event requires sharing a console with the child, and `WM_CLOSE`
11562/// requires the child to pump a message loop. A supervised server process does
11563/// neither, so there is nothing to send and teardown is the wait followed by the
11564/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11565/// the thing `protocol: "none"` exists to avoid.
11566#[cfg(not(unix))]
11567fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11568    debug!(
11569        module_id,
11570        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11571    );
11572}
11573
11574fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11575    match final_state {
11576        ModuleState::Stopped => TerminalDisposition::Stopped,
11577        ModuleState::Disabled => TerminalDisposition::Disabled,
11578        ModuleState::Restarting => TerminalDisposition::Restarting,
11579        ModuleState::Failed => TerminalDisposition::Failed,
11580        ModuleState::Starting
11581        | ModuleState::Running
11582        | ModuleState::Unresponsive
11583        | ModuleState::Draining => {
11584            unreachable!("terminal exits only finish in terminal or restarting states")
11585        }
11586    }
11587}
11588
11589/// Release a reaped child's registration before allowing another spawn.
11590///
11591/// EOF is not a process-lifetime signal: an inherited socket can stay open
11592/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11593/// reading EOF. After the normal release grace, request connection close (which
11594/// cancels both reads and dispatch), then allow one more release grace for the
11595/// connection guard's forwarding cleanup. Never evict a different connection.
11596async fn release_dead_registration(
11597    registry: &Registry,
11598    forwarding: Option<&ForwardingTable>,
11599    snapshot: &SharedSnapshot,
11600    module_id: &str,
11601    failure: &FailureContext,
11602) -> Result<(), SuperviseError> {
11603    let result = async {
11604        let registration = registry
11605            .get_module(module_id)
11606            .map_err(SuperviseError::Registry)?;
11607        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11608            Ok(()) => return Ok(()),
11609            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11610            Err(err) => return Err(err),
11611        }
11612        let pid = lock_snapshot(snapshot)?.reaped_pid;
11613        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11614            warn!(
11615                module_id,
11616                pid,
11617                connection_id = registration.connection_id.get(),
11618                "reaped module registration outlived release grace; closing dead connection"
11619            );
11620            forwarding.request_connection_close(
11621                registration.connection_id,
11622                CloseReason::new(
11623                    "supervised_process_reaped",
11624                    format!("module '{module_id}' pid {pid} exited"),
11625                ),
11626            );
11627            wait_for_slot_registration_release(
11628                registry,
11629                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11630                REGISTRY_RELEASE_TIMEOUT,
11631            )
11632            .await?;
11633        }
11634        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11635    }
11636    .await;
11637    if let Err(err) = &result {
11638        fail_snapshot(snapshot, Some(module_id), None, failure);
11639        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11640    }
11641    result
11642}
11643
11644/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11645/// plain stop or restart waits for before it spawns a replacement.
11646async fn wait_for_registration_release(
11647    registry: &Registry,
11648    module_id: &str,
11649    wait: Duration,
11650) -> Result<(), SuperviseError> {
11651    wait_for_slot_registration_release(
11652        registry,
11653        crate::registry::RegistrationSlot::Active(module_id),
11654        wait,
11655    )
11656    .await
11657}
11658
11659/// Wait for the registration in `slot` to go away.
11660///
11661/// Keyed on the slot rather than the bare module id because a successful swap
11662/// never empties the id's active slot (the promoted candidate is in it), so an
11663/// id-keyed wait for the incumbent's release would always time out. Draining a
11664/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11665/// incumbent's connection instead.
11666async fn wait_for_slot_registration_release(
11667    registry: &Registry,
11668    slot: crate::registry::RegistrationSlot<'_>,
11669    wait: Duration,
11670) -> Result<(), SuperviseError> {
11671    let deadline = Instant::now() + wait;
11672    let mut release_events = registration_release_events().subscribe();
11673    let still_active = |registration: &crate::registry::ModuleRegistration| {
11674        SuperviseError::RegistrationStillActive {
11675            module_id: registration.manifest.module_id.clone(),
11676            waited: wait,
11677        }
11678    };
11679    loop {
11680        let _observed_generation = *release_events.borrow_and_update();
11681        let Some(registration) = registry
11682            .registration(slot)
11683            .map_err(SuperviseError::Registry)?
11684        else {
11685            return Ok(());
11686        };
11687
11688        let now = Instant::now();
11689        if now >= deadline {
11690            return Err(still_active(&registration));
11691        }
11692
11693        let remaining = deadline.saturating_duration_since(now);
11694        match timeout(remaining, release_events.changed()).await {
11695            Ok(Ok(())) | Ok(Err(_)) => {}
11696            Err(_) => return Err(still_active(&registration)),
11697        }
11698    }
11699}
11700
11701#[cfg(test)]
11702mod slot_registration_wait_tests {
11703    use super::*;
11704    use crate::registry::{ConnectionId, RegistrationSlot};
11705    use subc_protocol::manifest::ModuleManifest;
11706
11707    #[tokio::test]
11708    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11709        let registry = Arc::new(Registry::default());
11710        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11711        let runtime = supervisor.runtime_config();
11712        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11713        let spec = ModuleSpec {
11714            module_id: "enable-stale-registration".to_string(),
11715            program: PathBuf::from("/missing/enable-retry-test"),
11716            args: Vec::new(),
11717            env: Vec::new(),
11718            reserved: false,
11719            reserved_prefixes: Vec::new(),
11720            protocol: ModuleProtocol::Subc,
11721            overlap: Default::default(),
11722        };
11723        let connection = ConnectionId::new(90);
11724        registry
11725            .register_with_control_ops(
11726                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11727                1,
11728                connection,
11729                Vec::new(),
11730            )
11731            .unwrap();
11732        let mut child = None;
11733        let err = set_child_enabled(
11734            &spec,
11735            &runtime,
11736            &registry,
11737            &supervisor.process_liveness,
11738            &snapshot,
11739            &mut child,
11740            true,
11741            &runtime.failure_context(false),
11742        )
11743        .await
11744        .unwrap_err();
11745        assert!(matches!(
11746            err,
11747            SuperviseError::RegistrationStillActive { .. }
11748        ));
11749        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11750        assert!(child.is_none());
11751        registry.deregister_connection(connection).unwrap();
11752        let err = set_child_enabled(
11753            &spec,
11754            &runtime,
11755            &registry,
11756            &supervisor.process_liveness,
11757            &snapshot,
11758            &mut child,
11759            true,
11760            &runtime.failure_context(false),
11761        )
11762        .await
11763        .unwrap_err();
11764        assert!(
11765            matches!(err, SuperviseError::Spawn { .. }),
11766            "second enable must attempt a spawn: {err}"
11767        );
11768        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11769    }
11770
11771    const INCUMBENT: u64 = 1;
11772    const CANDIDATE: u64 = 2;
11773
11774    fn swapped_registry() -> Arc<Registry> {
11775        let registry = Arc::new(Registry::default());
11776        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11777        registry
11778            .register_with_control_ops(
11779                manifest.clone(),
11780                1,
11781                ConnectionId::new(INCUMBENT),
11782                Vec::new(),
11783            )
11784            .unwrap();
11785        registry
11786            .register_candidate_with_control_ops(
11787                manifest,
11788                1,
11789                ConnectionId::new(CANDIDATE),
11790                Vec::new(),
11791            )
11792            .unwrap();
11793        registry
11794    }
11795
11796    /// After a promotion the id's active slot is held by the new process, so an
11797    /// id-keyed wait for the incumbent's release can never succeed; the
11798    /// connection-keyed wait completes as soon as the incumbent deregisters.
11799    #[tokio::test]
11800    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11801        let registry = swapped_registry();
11802        registry.promote_candidate("m").unwrap().unwrap();
11803
11804        assert!(matches!(
11805            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11806            Err(SuperviseError::RegistrationStillActive { .. })
11807        ));
11808
11809        // Still held while the incumbent's connection has not deregistered.
11810        assert!(matches!(
11811            wait_for_slot_registration_release(
11812                &registry,
11813                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11814                Duration::from_millis(50),
11815            )
11816            .await,
11817            Err(SuperviseError::RegistrationStillActive { .. })
11818        ));
11819
11820        let releaser = Arc::clone(&registry);
11821        let release = tokio::spawn(async move {
11822            sleep(Duration::from_millis(20)).await;
11823            releaser
11824                .deregister_connection(ConnectionId::new(INCUMBENT))
11825                .unwrap();
11826            notify_registration_release();
11827        });
11828        wait_for_slot_registration_release(
11829            &registry,
11830            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11831            Duration::from_secs(5),
11832        )
11833        .await
11834        .expect("the incumbent's own registration is released");
11835        release.await.unwrap();
11836        assert!(registry.get_module("m").unwrap().is_some());
11837    }
11838
11839    /// The candidate slot is waited on separately from the active slot: the
11840    /// incumbent's registration neither holds up nor stands in for it.
11841    #[tokio::test]
11842    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11843        let registry = swapped_registry();
11844        assert!(matches!(
11845            wait_for_slot_registration_release(
11846                &registry,
11847                RegistrationSlot::Candidate("m"),
11848                Duration::from_millis(50),
11849            )
11850            .await,
11851            Err(SuperviseError::RegistrationStillActive { .. })
11852        ));
11853        registry
11854            .deregister_connection(ConnectionId::new(CANDIDATE))
11855            .unwrap();
11856        wait_for_slot_registration_release(
11857            &registry,
11858            RegistrationSlot::Candidate("m"),
11859            Duration::from_millis(50),
11860        )
11861        .await
11862        .expect("a candidate slot with no candidate is released");
11863        assert!(registry
11864            .registration(RegistrationSlot::Active("m"))
11865            .unwrap()
11866            .is_some());
11867    }
11868}
11869
11870fn classify_exit(status: &ExitStatus) -> ExitReport {
11871    ExitReport {
11872        kind: if status.success() {
11873            ExitKind::Clean
11874        } else {
11875            ExitKind::Crash
11876        },
11877        code: status.code(),
11878        signal: exit_signal(status),
11879        at_ms: unix_ms_now(),
11880    }
11881}
11882
11883/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11884/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11885/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11886/// disposition still must be `Failed` so the terminal ring is not silently missing
11887/// an entry, matching what `fail_snapshot` records for this same arm.
11888fn wait_error_exit_report() -> ExitReport {
11889    ExitReport {
11890        kind: ExitKind::Crash,
11891        code: None,
11892        signal: None,
11893        at_ms: unix_ms_now(),
11894    }
11895}
11896
11897#[cfg(unix)]
11898fn exit_signal(status: &ExitStatus) -> Option<i32> {
11899    use std::os::unix::process::ExitStatusExt;
11900
11901    status.signal()
11902}
11903
11904#[cfg(not(unix))]
11905fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11906    None
11907}
11908
11909/// Give an operator-touched module its full crash budget back.
11910///
11911/// Named for the counter it used to zero; it now empties the in-window ring,
11912/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11913/// ledger of what happened survives every operator action.
11914fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11915    update_snapshot(snapshot, Some(module_id), |state| {
11916        state.clear_crash_restarts();
11917    })
11918}
11919
11920fn set_running(
11921    snapshot: &SharedSnapshot,
11922    child: &SupervisedChild,
11923    module_id: &str,
11924    spawn_events: &SpawnEventFeed,
11925) -> Result<(), SuperviseError> {
11926    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11927        module_id: Some(module_id.to_string()),
11928    })?;
11929    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11930    if std::mem::take(&mut state.coalesced_restart_pending) {
11931        let generation = state.spawn_generation;
11932        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11933    }
11934    state.drain_disposition_detail = None;
11935    state.spawn_failure = None;
11936    state.waiting_for_previous_pid = None;
11937    state.waiting_for_previous_detail = None;
11938    // Every caller of this is a plain spawn, which always uses the primary key;
11939    // a promoted swap candidate sets the flag itself after this returns.
11940    state.in_alternate_slot = false;
11941    state.configuration_updated_since_spawn = false;
11942    state.spawned_protocol = Some(child.protocol);
11943    state.state = ModuleState::Running;
11944    state.enabled = true;
11945    state.process_alive = true;
11946    state.pid = child.id();
11947    state.subtree_target = child.subtree_target.clone();
11948    // Probe samples belong to one process. Keep module-level escalation and
11949    // late-answer history, but never charge a new child for its predecessor's misses.
11950    state.health.status = SupervisorHealthStatus::Unknown;
11951    state.health.consecutive_failures = 0;
11952    state.health.detail = None;
11953    state.health.metrics = None;
11954    state.health.last_probe_ms = None;
11955    state.failed_retry.healthy_since = None;
11956    state.failed_retry.failed_at = None;
11957    state.failed_retry.running_since = Some(child.spawned_at);
11958    #[cfg(windows)]
11959    {
11960        state.pipe_consumed = child
11961            .nonce_delivery
11962            .as_ref()
11963            .map(|delivery| delivery.consumed());
11964    }
11965    #[cfg(target_os = "macos")]
11966    {
11967        state.report_ready = Some(Arc::clone(&child.report_ready));
11968    }
11969    state.spawned_at_ms = Some(child.spawned_at_ms);
11970    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11971    state.spawned_from = Some(child.spawned_from.clone());
11972    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11973    state.spawned_file_identity = child.spawned_file_identity.clone();
11974    state.process_start_time = child.process_start_time;
11975    Ok(())
11976}
11977
11978fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11979    state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
11980    state.failed_retry.healthy_since = None;
11981    state.failed_retry.running_since = None;
11982    state.waiting_for_previous_pid = None;
11983    state.waiting_for_previous_detail = None;
11984    state.process_alive = false;
11985    state.spawned_protocol = None;
11986    state.pid = None;
11987    #[cfg(windows)]
11988    {
11989        state.pipe_consumed = None;
11990    }
11991    #[cfg(target_os = "macos")]
11992    {
11993        state.report_ready = None;
11994    }
11995    state.spawned_at_ms = None;
11996    state.exec_outcome = None;
11997    state.spawned_from = None;
11998    state.launch_spec_digest = None;
11999    state.spawned_file_identity = None;
12000    state.process_start_time = None;
12001    state.deliberate_severance = None;
12002}
12003
12004#[cfg(test)]
12005fn record_deliberate_severance(
12006    snapshot: &SharedSnapshot,
12007    identity: ProcessIdentity,
12008) -> Result<(), SuperviseError> {
12009    update_snapshot(snapshot, None, |state| {
12010        state.deliberate_severance = Some(identity);
12011    })
12012}
12013
12014fn apply_deliberate_severance_marker(
12015    snapshot: &SharedSnapshot,
12016    exited_identity: Option<ProcessIdentity>,
12017    mut exit_report: ExitReport,
12018) -> ExitReport {
12019    let marker = lock_snapshot(snapshot)
12020        .ok()
12021        .and_then(|mut state| state.deliberate_severance.take());
12022    if marker.is_some() && marker == exited_identity {
12023        exit_report.kind = ExitKind::DeliberateSeverance;
12024    }
12025    exit_report
12026}
12027
12028fn classify_reaped_child_exit(
12029    snapshot: &SharedSnapshot,
12030    child: &SupervisedChild,
12031    status: &ExitStatus,
12032) -> ExitReport {
12033    let _ = update_snapshot(snapshot, None, |state| {
12034        state.reaped_pid = Some(child.pid);
12035        state.spawn_failure = child.spawn_failure.clone();
12036    });
12037    if child.spawn_failure.as_deref() == Some("registration_timeout") {
12038        registration_failure_exit_report(classify_exit(status))
12039    } else {
12040        apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12041    }
12042}
12043
12044fn fail_snapshot(
12045    snapshot: &SharedSnapshot,
12046    module_id: Option<&str>,
12047    last_exit: Option<ExitReport>,
12048    failure: &FailureContext,
12049) {
12050    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12051        state.state = ModuleState::Failed;
12052        clear_current_process_facts(state);
12053        if failure.may_retry() && state.enabled {
12054            state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12055        } else if failure.shutdown.is_set() {
12056            state.failed_retry.cancel();
12057        }
12058        if let Some(last_exit) = last_exit {
12059            state.last_exit = Some(last_exit);
12060        }
12061    }) {
12062        error!(error = %err, "failed to mark supervisor state failed");
12063    }
12064}
12065
12066fn update_snapshot(
12067    snapshot: &SharedSnapshot,
12068    module_id: Option<&str>,
12069    update: impl FnOnce(&mut SupervisorSnapshot),
12070) -> Result<(), SuperviseError> {
12071    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12072        module_id: module_id.map(ToOwned::to_owned),
12073    })?;
12074    update(&mut state);
12075    Ok(())
12076}
12077
12078const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12079
12080fn lock_snapshot_for_control<'a>(
12081    snapshot: &'a SharedSnapshot,
12082    module_id: &str,
12083    caller: &'static str,
12084) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12085    let started_at = Instant::now();
12086    let guard = lock_snapshot(snapshot)?;
12087    let waited = started_at.elapsed();
12088    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12089        warn!(
12090            module_id = %module_id,
12091            waited_ms = waited.as_millis() as u64,
12092            caller = %caller,
12093            "slow snapshot lock"
12094        );
12095    }
12096    Ok(guard)
12097}
12098
12099fn lock_snapshot(
12100    snapshot: &SharedSnapshot,
12101) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12102    snapshot
12103        .lock()
12104        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12105}
12106
12107#[cfg(test)]
12108mod terminal_history_tests {
12109    use std::{
12110        path::PathBuf,
12111        sync::Arc,
12112        time::{Duration, Instant},
12113    };
12114
12115    use tokio::{sync::mpsc, time::sleep};
12116
12117    use super::{
12118        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12119        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12120        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12121        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12122        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12123        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12124        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12125        SupervisorHealthStatus, SupervisorSnapshot,
12126    };
12127    // The supervisor's clock, distinct from the `std::time::Instant` these tests
12128    // use for their own wall-clock deadlines: crash-restart instants must be on
12129    // the same clock the production code stamps them with, which is tokio's (and
12130    // is what `start_paused` tests can move).
12131    use super::Instant as ClockInstant;
12132    use crate::{
12133        registry::Registry,
12134        terminal_ring::{TerminalRing, TerminalRingConfig},
12135    };
12136    use std::sync::Mutex;
12137    use subc_control::TerminalDisposition;
12138
12139    /// See the twin in `control.rs` for why this derives the path from
12140    /// `current_exe()` and why the existence check is here: `--lib` alone does
12141    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
12142    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
12143    pub(super) fn fake_aft_stub_path() -> PathBuf {
12144        let mut path = std::env::current_exe().expect("current_exe available in tests");
12145        path.pop();
12146        path.pop();
12147        path.push(if cfg!(windows) {
12148            "fake-aft-stub.exe"
12149        } else {
12150            "fake-aft-stub"
12151        });
12152        assert!(
12153            path.exists(),
12154            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12155             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12156            path.display()
12157        );
12158        path
12159    }
12160
12161    #[test]
12162    fn reserved_never_spawned_refuses_every_hello() {
12163        // The canary hole: a reserved id whose module has never spawned had NO
12164        // gate entry and admitted anyone -- the reservation protected the nonce
12165        // holder, not the NAME. Now the entry is present with no legitimate
12166        // holder and refuses all comers.
12167        let supervisor = SupervisorHandle::default();
12168        supervisor.apply_identity_configuration(&ModuleSpec {
12169            module_id: "never-spawned".to_string(),
12170            program: PathBuf::from("/usr/bin/false"),
12171            args: Vec::new(),
12172            env: Vec::new(),
12173            reserved: true,
12174            reserved_prefixes: Vec::new(),
12175            protocol: ModuleProtocol::Subc,
12176            overlap: Default::default(),
12177        });
12178        assert!(
12179            supervisor
12180                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12181                .is_err(),
12182            "forged nonce must refuse on a reserved never-spawned id"
12183        );
12184        assert!(
12185            supervisor
12186                .reserved_hello_verification("never-spawned", None)
12187                .is_err(),
12188            "absent nonce must refuse on a reserved never-spawned id"
12189        );
12190        // And a real spawn nonce minted later admits exactly that nonce.
12191        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12192        supervisor.apply_identity_configuration(&ModuleSpec {
12193            module_id: "never-spawned".to_string(),
12194            program: PathBuf::from("/usr/bin/false"),
12195            args: Vec::new(),
12196            env: Vec::new(),
12197            reserved: true,
12198            reserved_prefixes: Vec::new(),
12199            protocol: ModuleProtocol::Subc,
12200            overlap: Default::default(),
12201        });
12202        assert!(supervisor
12203            .reserved_hello_verification("never-spawned", Some("minted"))
12204            .is_ok());
12205        assert!(supervisor
12206            .reserved_hello_verification("never-spawned", Some("forged"))
12207            .is_err());
12208    }
12209
12210    /// Put `count` crash restarts on a snapshot's ring as if they had all just
12211    /// happened, which is what "spent budget" looks like to every reader.
12212    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12213        let now = ClockInstant::now();
12214        for _ in 0..count {
12215            state.crash_restarts.push_back(now);
12216        }
12217    }
12218
12219    /// Age the oldest recorded restart out of `window`, standing in for the hours
12220    /// that would otherwise have to pass. Injecting the instant is the point: a
12221    /// test that slept a real window would take ten minutes and still prove less.
12222    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12223        let aged = state
12224            .crash_restarts
12225            .front()
12226            .expect("a crash restart must be recorded before it can be aged")
12227            .checked_sub(window + Duration::from_secs(1))
12228            .expect("the test clock is far enough from its origin to age an instant");
12229        state.crash_restarts[0] = aged;
12230    }
12231
12232    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12233        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12234        seed_crash_restarts(&mut state, count);
12235        state
12236    }
12237
12238    #[test]
12239    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12240        let policy = RestartPolicy::new(3, Duration::ZERO);
12241        let now = ClockInstant::now();
12242        assert!(daemon_will_restart(
12243            &mut snapshot_with_restarts(true, 2),
12244            &policy,
12245            now
12246        ));
12247        assert!(!daemon_will_restart(
12248            &mut snapshot_with_restarts(true, 3),
12249            &policy,
12250            now
12251        ));
12252        assert!(!daemon_will_restart(
12253            &mut snapshot_with_restarts(false, 0),
12254            &policy,
12255            now
12256        ));
12257    }
12258
12259    #[test]
12260    fn crash_restart_backoff_escalates_with_in_window_count() {
12261        let policy = RestartPolicy::new(4, Duration::from_millis(100))
12262            .with_max_backoff(Duration::from_secs(30));
12263        let now = ClockInstant::now();
12264        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12265        let schedules = (0..4)
12266            .map(|_| {
12267                state
12268                    .next_crash_restart(&policy, now)
12269                    .expect("the test policy allows four crash restarts")
12270            })
12271            .collect::<Vec<_>>();
12272
12273        assert_eq!(
12274            schedules
12275                .iter()
12276                .map(|schedule| schedule.restart_in_window)
12277                .collect::<Vec<_>>(),
12278            vec![0, 1, 2, 3]
12279        );
12280        assert_eq!(
12281            schedules
12282                .iter()
12283                .map(|schedule| schedule.delay)
12284                .collect::<Vec<_>>(),
12285            vec![
12286                Duration::from_millis(100),
12287                Duration::from_secs(1),
12288                Duration::from_secs(10),
12289                Duration::from_secs(30),
12290            ]
12291        );
12292    }
12293
12294    #[test]
12295    fn crash_restart_backoff_resets_after_ring_clear() {
12296        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12297        let now = ClockInstant::now();
12298        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12299        assert_eq!(
12300            state.next_crash_restart(&policy, now).unwrap().delay,
12301            Duration::from_millis(100)
12302        );
12303        assert_eq!(
12304            state.next_crash_restart(&policy, now).unwrap().delay,
12305            Duration::from_secs(1)
12306        );
12307
12308        state.clear_crash_restarts();
12309        let schedule = state
12310            .next_crash_restart(&policy, now)
12311            .expect("a cleared ring must allow another restart");
12312        assert_eq!(schedule.restart_in_window, 0);
12313        assert_eq!(schedule.delay, Duration::from_millis(100));
12314    }
12315
12316    #[test]
12317    fn crash_restart_backoff_ignores_aged_restarts() {
12318        let policy = RestartPolicy::new(3, Duration::from_millis(100));
12319        let now = ClockInstant::now();
12320        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12321        state
12322            .next_crash_restart(&policy, now)
12323            .expect("the first restart is allowed");
12324        state
12325            .next_crash_restart(&policy, now)
12326            .expect("the second restart is allowed");
12327        state.crash_restarts[0] = now
12328            .checked_sub(policy.window + Duration::from_secs(1))
12329            .expect("the fake clock can age a restart past the window");
12330
12331        let schedule = state
12332            .next_crash_restart(&policy, now)
12333            .expect("an aged restart must release its slot");
12334        assert_eq!(schedule.restart_in_window, 1);
12335        assert_eq!(schedule.delay, Duration::from_secs(1));
12336        assert_eq!(state.crash_restarts.len(), 2);
12337    }
12338
12339    /// The budget is a rate: the same three spent restarts refuse a respawn
12340    /// while they are recent and allow one once they have aged past the window.
12341    /// Nothing about the module changed in between, which is the whole point.
12342    #[test]
12343    fn a_budget_spent_before_the_window_no_longer_refuses() {
12344        let policy = RestartPolicy::new(3, Duration::ZERO);
12345        let mut state = snapshot_with_restarts(true, 3);
12346        let now = ClockInstant::now();
12347        assert!(!daemon_will_restart(&mut state, &policy, now));
12348
12349        assert!(daemon_will_restart(
12350            &mut state,
12351            &policy,
12352            now + policy.window + Duration::from_secs(1)
12353        ));
12354        assert!(
12355            state.crash_restarts.is_empty(),
12356            "reading the budget must drop the instants that left the window"
12357        );
12358    }
12359
12360    fn module_with_recovery_snapshot(
12361        state: ModuleState,
12362        enabled: bool,
12363        restart_count: u32,
12364    ) -> SupervisedModule {
12365        let registry = Arc::new(Registry::default());
12366        let supervisor =
12367            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
12368        let runtime = supervisor.runtime_config();
12369        let spec = ModuleSpec {
12370            module_id: "recovery-snapshot".to_string(),
12371            program: fake_aft_stub_path(),
12372            args: Vec::new(),
12373            env: Vec::new(),
12374            reserved: false,
12375            reserved_prefixes: Vec::new(),
12376            protocol: ModuleProtocol::Subc,
12377            overlap: Default::default(),
12378        };
12379        let mut snapshot = SupervisorSnapshot::new(state, enabled);
12380        seed_crash_restarts(&mut snapshot, restart_count);
12381        // These tests read synthetic snapshots. A real child and monitor would
12382        // race those reads by replacing the requested state during startup.
12383        let (commands, _rx) = mpsc::channel(4);
12384        SupervisedModule {
12385            inner: Arc::new(SupervisedModuleInner {
12386                module_id: spec.module_id.clone(),
12387                registry,
12388                snapshot: Arc::new(Mutex::new(snapshot)),
12389                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12390                    spec,
12391                    health: runtime.health,
12392                })),
12393                stderr_ring: runtime.stderr_ring,
12394                terminal_ring: runtime.terminal_ring,
12395                commands,
12396                monitor: Mutex::new(None),
12397                restart_policy: runtime.restart_policy,
12398                effective_drain_timeout: runtime.effective_drain_timeout,
12399                provenance_probe: supervisor.provenance_probe.clone(),
12400            }),
12401        }
12402    }
12403
12404    #[cfg(target_os = "linux")]
12405    #[tokio::test]
12406    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12407        let supervisor =
12408            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12409                .with_cgroup_placement(None);
12410        let result = supervisor.spawn(ModuleSpec {
12411            module_id: "no-cgroup-placement".to_string(),
12412            program: fake_aft_stub_path(),
12413            args: Vec::new(),
12414            env: Vec::new(),
12415            reserved: false,
12416            reserved_prefixes: Vec::new(),
12417            protocol: ModuleProtocol::Subc,
12418            overlap: Default::default(),
12419        });
12420
12421        assert!(
12422            result.is_ok(),
12423            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12424        );
12425    }
12426
12427    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12428    async fn undecided_snapshot_uses_shared_restart_predicate() {
12429        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12430            .will_recover_after_connection_loss()
12431            .unwrap());
12432        assert!(
12433            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12434                .will_recover_after_connection_loss()
12435                .unwrap()
12436        );
12437    }
12438
12439    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12440    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12441        assert!(
12442            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12443                .will_recover_after_connection_loss()
12444                .unwrap()
12445        );
12446    }
12447
12448    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12449    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12450        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12451            .will_recover_after_connection_loss()
12452            .unwrap());
12453        assert!(
12454            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12455                .will_recover_after_connection_loss()
12456                .unwrap()
12457        );
12458    }
12459
12460    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12461    async fn warming_snapshot_is_limited_to_startup_phases() {
12462        for state in [
12463            ModuleState::Starting,
12464            ModuleState::Running,
12465            ModuleState::Restarting,
12466        ] {
12467            assert!(
12468                module_with_recovery_snapshot(state, true, 0)
12469                    .is_warming()
12470                    .unwrap(),
12471                "{state:?} should be warming"
12472            );
12473        }
12474        for state in [
12475            ModuleState::Unresponsive,
12476            ModuleState::Draining,
12477            ModuleState::Stopped,
12478            ModuleState::Failed,
12479            ModuleState::Disabled,
12480        ] {
12481            assert!(
12482                !module_with_recovery_snapshot(state, true, 0)
12483                    .is_warming()
12484                    .unwrap(),
12485                "{state:?} should not be warming"
12486            );
12487        }
12488    }
12489
12490    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12491    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12492        let registry = Arc::new(Registry::default());
12493        let supervisor =
12494            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
12495        let module = supervisor
12496            .spawn(ModuleSpec {
12497                module_id: "terminal-history".to_string(),
12498                program: fake_aft_stub_path(),
12499                args: Vec::new(),
12500                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12501                reserved: false,
12502                reserved_prefixes: Vec::new(),
12503                protocol: ModuleProtocol::Subc,
12504                overlap: Default::default(),
12505            })
12506            .unwrap();
12507
12508        let deadline = Instant::now() + Duration::from_secs(5);
12509        loop {
12510            let history = module.terminal_history();
12511            if history.entries.len() == 2 {
12512                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12513                assert_eq!(history.dropped, 0);
12514                assert_eq!(
12515                    history
12516                        .entries
12517                        .iter()
12518                        .map(|entry| entry.exit_code)
12519                        .collect::<Vec<_>>(),
12520                    vec![Some(23), Some(23)]
12521                );
12522                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12523                return;
12524            }
12525            assert!(
12526                Instant::now() < deadline,
12527                "module did not retain two terminal exits: {history:?}"
12528            );
12529            sleep(Duration::from_millis(10)).await;
12530        }
12531    }
12532
12533    /// A disable issued while a crash respawn is still backing off must preempt
12534    /// that respawn: the operator's stop wins, the disable must not queue behind
12535    /// the backoff, and the module must never come back up afterwards.
12536    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12537    async fn disable_during_crash_backoff_cancels_pending_respawn() {
12538        let backoff = Duration::from_secs(2);
12539        let supervisor = Supervisor::new_for_test(
12540            Arc::new(Registry::default()),
12541            RestartPolicy::new(10, backoff),
12542        );
12543        let module = supervisor
12544            .spawn(ModuleSpec {
12545                module_id: "disable-during-backoff".to_string(),
12546                program: fake_aft_stub_path(),
12547                args: Vec::new(),
12548                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12549                reserved: false,
12550                reserved_prefixes: Vec::new(),
12551                protocol: ModuleProtocol::Subc,
12552                overlap: Default::default(),
12553            })
12554            .unwrap();
12555
12556        // Wait for the first crash to put the module into its backoff window.
12557        let deadline = Instant::now() + Duration::from_secs(5);
12558        loop {
12559            if module.status().unwrap().state == ModuleState::Restarting {
12560                break;
12561            }
12562            assert!(
12563                Instant::now() < deadline,
12564                "module never entered the crash backoff"
12565            );
12566            sleep(Duration::from_millis(10)).await;
12567        }
12568
12569        let started = Instant::now();
12570        module.set_enabled(false).await.unwrap();
12571        let waited = started.elapsed();
12572
12573        assert!(
12574            waited < backoff / 2,
12575            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12576        );
12577        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12578
12579        // Outlast the backoff: the respawn it was counting down to must never run.
12580        sleep(backoff + Duration::from_millis(500)).await;
12581        let status = module.status().unwrap();
12582        assert_eq!(status.state, ModuleState::Disabled);
12583        assert_eq!(
12584            status.spawn_generation, 1,
12585            "module respawned after the operator disabled it"
12586        );
12587    }
12588
12589    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12590    /// the shape of nats-server, the program this rule exists for.
12591    #[cfg(unix)]
12592    fn protocol_none_sigterm_exits_clean_spec(
12593        module_id: &str,
12594        dir: &std::path::Path,
12595    ) -> (ModuleSpec, PathBuf, PathBuf) {
12596        let ready = dir.join("ready");
12597        let marker = dir.join("sigterm");
12598        let spec = ModuleSpec {
12599            module_id: module_id.to_string(),
12600            program: fake_aft_stub_path(),
12601            args: Vec::new(),
12602            env: vec![
12603                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12604                (
12605                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12606                    marker.display().to_string(),
12607                ),
12608                (
12609                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12610                    ready.display().to_string(),
12611                ),
12612            ],
12613            reserved: false,
12614            reserved_prefixes: Vec::new(),
12615            protocol: ModuleProtocol::None,
12616            overlap: Default::default(),
12617        };
12618        (spec, ready, marker)
12619    }
12620
12621    /// Wait for a file the child writes, so a signal is never sent before the
12622    /// child's SIGTERM handler is installed (the default disposition would
12623    /// kill it by signal and the exit would not be clean).
12624    #[cfg(unix)]
12625    async fn wait_for_file(path: &std::path::Path) {
12626        let deadline = Instant::now() + Duration::from_secs(10);
12627        while !path.exists() {
12628            assert!(
12629                Instant::now() < deadline,
12630                "{} never appeared",
12631                path.display()
12632            );
12633            sleep(Duration::from_millis(10)).await;
12634        }
12635    }
12636
12637    /// A protocol-none module that exits 0 because something OUTSIDE the
12638    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12639    /// the crash-path disposition rather than `stopped`.
12640    #[cfg(unix)]
12641    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12642    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12643        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12644        let (spec, ready, marker) =
12645            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12646        let supervisor = Supervisor::new_for_test(
12647            Arc::new(Registry::default()),
12648            RestartPolicy::new(3, Duration::ZERO),
12649        );
12650        let module = supervisor.spawn(spec).unwrap();
12651        wait_for_file(&ready).await;
12652        // The ready file proves the child installed its SIGTERM handler, not
12653        // that the supervisor has processed the privacy trampoline's exec
12654        // acknowledgement. On macOS status withholds the pid until then.
12655        let deadline = Instant::now() + Duration::from_secs(10);
12656        let first_pid = loop {
12657            let status = module.status().unwrap();
12658            if status.state == ModuleState::Running {
12659                if let Some(pid) = status.pid {
12660                    break pid;
12661                }
12662            }
12663            assert!(
12664                Instant::now() < deadline,
12665                "a running module must report its pid after exec confirmation: {status:?}"
12666            );
12667            sleep(Duration::from_millis(10)).await;
12668        };
12669
12670        rustix::process::kill_process(
12671            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12672            rustix::process::Signal::TERM,
12673        )
12674        .unwrap();
12675
12676        let deadline = Instant::now() + Duration::from_secs(10);
12677        let respawned = loop {
12678            let status = module.status().unwrap();
12679            if status.state == ModuleState::Running
12680                && status.pid.is_some_and(|pid| pid != first_pid)
12681            {
12682                break status;
12683            }
12684            assert!(
12685                Instant::now() < deadline,
12686                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12687            );
12688            sleep(Duration::from_millis(10)).await;
12689        };
12690        assert_eq!(respawned.spawn_generation, 2);
12691        assert!(
12692            marker.exists(),
12693            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12694        );
12695
12696        let history = module.terminal_history();
12697        assert_eq!(history.entries.len(), 1, "{history:?}");
12698        let entry = &history.entries[0];
12699        assert_eq!(entry.exit_code, Some(0));
12700        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12701        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12702
12703        module.stop().await.unwrap();
12704    }
12705
12706    /// Repeated unrequested clean exits of a protocol-none module spend the
12707    /// restart budget exactly as crashes do, and the module ends `failed` with
12708    /// the budget named.
12709    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12710    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12711        let supervisor = Supervisor::new_for_test(
12712            Arc::new(Registry::default()),
12713            RestartPolicy::new(1, Duration::ZERO),
12714        );
12715        let module = supervisor
12716            .spawn(ModuleSpec {
12717                module_id: "none-clean-exit-budget".to_string(),
12718                program: fake_aft_stub_path(),
12719                args: Vec::new(),
12720                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12721                reserved: false,
12722                reserved_prefixes: Vec::new(),
12723                protocol: ModuleProtocol::None,
12724                overlap: Default::default(),
12725            })
12726            .unwrap();
12727
12728        // Failed follows the terminal write; this deadline only bounds a hang,
12729        // not an assumed duration for the two launches or their exit recording.
12730        let deadline = Instant::now() + Duration::from_secs(10);
12731        loop {
12732            let status = module.status().unwrap();
12733            if status.state == ModuleState::Failed {
12734                break;
12735            }
12736            assert!(
12737                Instant::now() < deadline,
12738                "module never exhausted its budget: {status:?} {:?}",
12739                module.terminal_history()
12740            );
12741            sleep(Duration::from_millis(10)).await;
12742        }
12743        let history = module.terminal_history();
12744        assert_eq!(
12745            history
12746                .entries
12747                .iter()
12748                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12749                .collect::<Vec<_>>(),
12750            vec![
12751                (Some(0), TerminalDisposition::Restarting),
12752                (Some(0), TerminalDisposition::Failed),
12753            ]
12754        );
12755        let detail = history.entries[1]
12756            .disposition_detail
12757            .as_deref()
12758            .expect("a budget failure names the budget");
12759        assert!(detail.contains("max_restarts=1"), "{detail}");
12760        assert_eq!(module.status().unwrap().spawn_generation, 2);
12761    }
12762
12763    #[test]
12764    fn restart_budget_failure_is_published_after_its_terminal_record() {
12765        use std::io::{Read, Write};
12766        let supervisor = Supervisor::new_for_test(
12767            Arc::new(Registry::default()),
12768            RestartPolicy::new(0, Duration::ZERO),
12769        );
12770        let runtime = supervisor.runtime_config();
12771        let spec = ModuleSpec {
12772            protocol: ModuleProtocol::None,
12773            ..windowed_crash_spec("budget-publication-order")
12774        };
12775        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12776            ModuleState::Running,
12777            true,
12778        )));
12779        lock_snapshot(&snapshot).unwrap().process_alive = true;
12780        let reaped_snapshot = snapshot.clone();
12781        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12782        let history = runtime.terminal_ring.clone();
12783        let reap = std::thread::spawn(move || {
12784            tokio::runtime::Builder::new_current_thread()
12785                .enable_all()
12786                .build()
12787                .unwrap()
12788                .block_on(on_child_exit(
12789                    &spec,
12790                    runtime.restart_policy,
12791                    &supervisor.registry,
12792                    &reaped_snapshot,
12793                    &runtime.terminal_ring,
12794                    &runtime.spawn_events,
12795                    &runtime.child_roster,
12796                    ExitReport {
12797                        kind: ExitKind::Clean,
12798                        code: Some(0),
12799                        signal: None,
12800                        at_ms: 1,
12801                    },
12802                    true,
12803                ))
12804        });
12805        let mut reached = [0];
12806        barrier.read_exact(&mut reached).unwrap();
12807        assert_eq!(&reached, b"R");
12808        let (before_state, before_alive) = {
12809            let state = lock_snapshot(&snapshot).unwrap();
12810            (state.state, state.process_alive)
12811        };
12812        let before_history = history.lock().unwrap().snapshot();
12813        barrier.write_all(b"X").unwrap();
12814        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12815        assert!(before_history.entries.is_empty());
12816        assert_ne!(
12817            before_state,
12818            ModuleState::Failed,
12819            "Failed was visible before its terminal record could be written"
12820        );
12821        assert!(before_alive);
12822        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12823        let history = history.lock().unwrap().snapshot();
12824        assert_eq!(history.entries.len(), 1);
12825        assert_eq!(history.entries[0].exit_code, Some(0));
12826        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12827    }
12828
12829    #[test]
12830    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12831        use std::io::{Read, Write};
12832
12833        let supervisor = Supervisor::new_for_test(
12834            Arc::new(Registry::default()),
12835            RestartPolicy::new(3, Duration::ZERO),
12836        );
12837        let mut runtime = supervisor.runtime_config();
12838        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12839        let snapshot = module.inner.snapshot.clone();
12840        lock_snapshot(&snapshot).unwrap().process_alive = true;
12841        runtime.terminal_ring = module.inner.terminal_ring.clone();
12842        let spec = windowed_crash_spec(module.module_id());
12843        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12844        let reaped_snapshot = snapshot.clone();
12845        let reap = std::thread::spawn(move || {
12846            tokio::runtime::Builder::new_current_thread()
12847                .enable_all()
12848                .build()
12849                .unwrap()
12850                .block_on(on_child_exit(
12851                    &spec,
12852                    runtime.restart_policy,
12853                    &supervisor.registry,
12854                    &reaped_snapshot,
12855                    &runtime.terminal_ring,
12856                    &runtime.spawn_events,
12857                    &runtime.child_roster,
12858                    ExitReport {
12859                        kind: ExitKind::Crash,
12860                        code: Some(2),
12861                        signal: None,
12862                        at_ms: 1,
12863                    },
12864                    true,
12865                ))
12866        });
12867
12868        // Hold the crash handler between its budget decision and publication.
12869        // Connection cleanup must still see the recovery that this crash chose,
12870        // even when that recovery consumes the last available restart slot.
12871        let mut reached = [0];
12872        barrier.read_exact(&mut reached).unwrap();
12873        assert_eq!(&reached, b"R");
12874        let before_status = module.status().unwrap();
12875        let before_history = module.terminal_history();
12876        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12877        barrier.write_all(b"X").unwrap();
12878        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12879
12880        assert_eq!(before_status.state, ModuleState::Running);
12881        assert!(before_status.process_alive);
12882        assert_eq!(before_status.restart_count, 3);
12883        assert!(before_history.entries.is_empty());
12884        assert!(
12885            before_recovers,
12886            "connection cleanup would report a terminal close for a scheduled crash restart"
12887        );
12888        let restarting = module.status().unwrap();
12889        assert_eq!(restarting.state, ModuleState::Restarting);
12890        assert_eq!(restarting.restart_count, 3);
12891        assert!(module.will_recover_after_connection_loss().unwrap());
12892        let history = module.terminal_history();
12893        assert_eq!(history.entries.len(), 1);
12894        assert_eq!(
12895            history.entries[0].disposition,
12896            TerminalDisposition::Restarting
12897        );
12898
12899        // Once the replacement is running, its next crash must be judged
12900        // against the exhausted budget rather than the previous recovery.
12901        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12902            state.state = ModuleState::Running;
12903            state.respawn_pending = false;
12904        })
12905        .unwrap();
12906        assert!(!module.will_recover_after_connection_loss().unwrap());
12907        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12908            state.enabled = false;
12909            state.respawn_pending = true;
12910        })
12911        .unwrap();
12912        assert!(!module.will_recover_after_connection_loss().unwrap());
12913    }
12914
12915    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12916        use std::net::{TcpListener, TcpStream};
12917        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12918        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12919        let (barrier, _) = listener.accept().unwrap();
12920        for stream in [&writer, &barrier] {
12921            stream
12922                .set_read_timeout(Some(Duration::from_secs(10)))
12923                .unwrap();
12924            stream
12925                .set_write_timeout(Some(Duration::from_secs(10)))
12926                .unwrap();
12927        }
12928        *events.1.lock().unwrap() = Some(writer);
12929        barrier
12930    }
12931
12932    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12933        use std::io::{Read, Write};
12934        let barrier = events.1.lock().unwrap().take();
12935        if let Some(mut barrier) = barrier {
12936            barrier.write_all(b"R").unwrap();
12937            let mut release = [0];
12938            barrier.read_exact(&mut release).unwrap();
12939            assert_eq!(&release, b"X");
12940        }
12941    }
12942
12943    #[test]
12944    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12945        use std::io::{Read, Write};
12946
12947        let supervisor = Supervisor::new_for_test(
12948            Arc::new(Registry::default()),
12949            RestartPolicy::new(0, Duration::ZERO),
12950        );
12951        let mut runtime = supervisor.runtime_config();
12952        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12953        let snapshot = module.inner.snapshot.clone();
12954        runtime.terminal_ring = module.inner.terminal_ring.clone();
12955        let spec = ModuleSpec {
12956            module_id: module.module_id().into(),
12957            program: fake_aft_stub_path(),
12958            protocol: ModuleProtocol::None,
12959            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12960            ..windowed_crash_spec(module.module_id())
12961        };
12962        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12963        let reap = std::thread::spawn(move || {
12964            tokio::runtime::Builder::new_current_thread()
12965                .enable_all()
12966                .build()
12967                .unwrap()
12968                .block_on(async {
12969                    let mut child =
12970                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12971                    health_restart_child(
12972                        &spec,
12973                        &runtime,
12974                        &supervisor.registry,
12975                        &supervisor.process_liveness,
12976                        &snapshot,
12977                        &mut child,
12978                        SupervisorHealthStatus::Failing,
12979                        Some("health probe did not answer"),
12980                        1,
12981                    )
12982                    .await
12983                    .unwrap();
12984                    assert!(child.is_none());
12985                });
12986        });
12987        // The loopback barrier acknowledges entry into terminal recording by
12988        // health_restart_child's drain, then waits for the test's release byte.
12989        // Status and history are observed while that writer is held, not after
12990        // an elapsed-time guess about when the health action has completed.
12991        let mut reached = [0];
12992        barrier.read_exact(&mut reached).unwrap();
12993        assert_eq!(&reached, b"R");
12994        let before_status = module.status().unwrap();
12995        let before_history = module.terminal_history();
12996        barrier.write_all(b"X").unwrap();
12997        reap.join().unwrap();
12998        assert!(before_history.entries.is_empty());
12999        assert_ne!(
13000            before_status.state,
13001            ModuleState::Failed,
13002            "Failed was visible before its terminal record could be written"
13003        );
13004        assert!(
13005            before_status.process_alive,
13006            "process death was visible before its terminal record could be written"
13007        );
13008        let failed = module.status().unwrap();
13009        assert_eq!(failed.state, ModuleState::Failed);
13010        assert!(!failed.process_alive);
13011        let history = module.terminal_history();
13012        assert_eq!(history.entries.len(), 1);
13013        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13014        assert_eq!(
13015            history.entries[0].disposition_detail.as_deref(),
13016            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13017        );
13018    }
13019
13020    #[test]
13021    fn observed_exits_publish_process_death_after_terminal_history() {
13022        use std::io::{Read, Write};
13023        for (kind, enabled, shutdown, wait_error, expected_state) in [
13024            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13025            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13026            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13027            (
13028                ExitKind::DeliberateSeverance,
13029                true,
13030                false,
13031                false,
13032                ModuleState::Restarting,
13033            ),
13034            #[cfg(unix)]
13035            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13036            (ExitKind::Crash, true, false, true, ModuleState::Failed),
13037        ] {
13038            let supervisor = Supervisor::new_for_test(
13039                Arc::new(Registry::default()),
13040                RestartPolicy::new(1, Duration::ZERO),
13041            );
13042            let runtime = supervisor.runtime_config();
13043            #[cfg(unix)]
13044            if shutdown {
13045                runtime.child_roster.close();
13046            }
13047            #[cfg(not(unix))]
13048            let _ = shutdown;
13049            let spec = windowed_crash_spec("exit-publication-order");
13050            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13051                ModuleState::Running,
13052                enabled,
13053            )));
13054            lock_snapshot(&snapshot).unwrap().process_alive = true;
13055            let reaped_snapshot = snapshot.clone();
13056            let history = runtime.terminal_ring.clone();
13057            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13058            let reap = std::thread::spawn(move || {
13059                if wait_error {
13060                    super::fail_after_wait_error(
13061                        &spec.module_id,
13062                        &reaped_snapshot,
13063                        &runtime.terminal_ring,
13064                        &runtime.spawn_events,
13065                        "wait failed".into(),
13066                        &runtime.failure_context(true),
13067                    );
13068                } else {
13069                    tokio::runtime::Builder::new_current_thread()
13070                        .enable_all()
13071                        .build()
13072                        .unwrap()
13073                        .block_on(on_child_exit(
13074                            &spec,
13075                            runtime.restart_policy,
13076                            &supervisor.registry,
13077                            &reaped_snapshot,
13078                            &runtime.terminal_ring,
13079                            &runtime.spawn_events,
13080                            &runtime.child_roster,
13081                            ExitReport {
13082                                kind,
13083                                code: Some(0),
13084                                signal: None,
13085                                at_ms: 1,
13086                            },
13087                            true,
13088                        ));
13089                }
13090            });
13091            let mut reached = [0];
13092            barrier.read_exact(&mut reached).unwrap();
13093            assert_eq!(&reached, b"R");
13094            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13095            let before_history = history.lock().unwrap().snapshot();
13096            barrier.write_all(b"X").unwrap();
13097            reap.join().unwrap();
13098            assert!(
13099                before_alive,
13100                "{kind:?}: death was published before recording"
13101            );
13102            assert!(before_history.entries.is_empty());
13103            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13104            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13105            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13106        }
13107    }
13108
13109    #[tokio::test]
13110    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13111        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13112        let path = dir.join("terminals.jsonl");
13113        std::fs::create_dir(&path).unwrap();
13114        let supervisor = Supervisor::new_for_test(
13115            Arc::new(Registry::default()),
13116            RestartPolicy::new(0, Duration::ZERO),
13117        )
13118        .with_terminal_journal(path, "health-budget-journal-failure".into());
13119        let runtime = supervisor.runtime_config();
13120        let spec = ModuleSpec {
13121            program: fake_aft_stub_path(),
13122            protocol: ModuleProtocol::None,
13123            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13124            ..windowed_crash_spec("health-budget-journal-failure")
13125        };
13126        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13127        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13128        health_restart_child(
13129            &spec,
13130            &runtime,
13131            &supervisor.registry,
13132            &supervisor.process_liveness,
13133            &snapshot,
13134            &mut child,
13135            SupervisorHealthStatus::Failing,
13136            None,
13137            1,
13138        )
13139        .await
13140        .unwrap();
13141        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13142        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13143        let history = runtime
13144            .terminal_ring
13145            .lock()
13146            .unwrap()
13147            .durable_history(&spec.module_id);
13148        assert!(history.journal_write_failures > 0);
13149        assert_eq!(history.entries.len(), 1);
13150        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13151    }
13152
13153    #[test]
13154    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13155        let supervisor = Supervisor::new_for_test(
13156            Arc::new(Registry::default()),
13157            RestartPolicy::new(0, Duration::ZERO),
13158        );
13159        let runtime = supervisor.runtime_config();
13160        let spec = ModuleSpec {
13161            program: fake_aft_stub_path(),
13162            protocol: ModuleProtocol::None,
13163            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13164            ..windowed_crash_spec("health-budget-recording-panic")
13165        };
13166        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13167        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13168            tokio::runtime::Builder::new_current_thread()
13169                .enable_all()
13170                .build()
13171                .unwrap()
13172                .block_on(async {
13173                    let mut child =
13174                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13175                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13176                    health_restart_child(
13177                        &spec,
13178                        &runtime,
13179                        &supervisor.registry,
13180                        &supervisor.process_liveness,
13181                        &snapshot,
13182                        &mut child,
13183                        SupervisorHealthStatus::Failing,
13184                        None,
13185                        1,
13186                    )
13187                    .await
13188                    .unwrap();
13189                });
13190        }));
13191        let panic = result.expect_err("recording must still unwind");
13192        assert_eq!(
13193            panic.downcast_ref::<String>().map(String::as_str),
13194            Some("spawn event sequence exhausted")
13195        );
13196        let state = lock_snapshot(&snapshot).unwrap();
13197        assert_eq!(state.state, ModuleState::Failed);
13198        assert!(!state.process_alive);
13199        assert!(state.last_exit.is_some());
13200    }
13201
13202    #[tokio::test]
13203    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13204        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13205        let path = dir.join("terminals.jsonl");
13206        std::fs::create_dir(&path).unwrap();
13207        let supervisor = Supervisor::new_for_test(
13208            Arc::new(Registry::default()),
13209            RestartPolicy::new(0, Duration::ZERO),
13210        )
13211        .with_terminal_journal(path, "budget-journal-failure".into());
13212        let runtime = supervisor.runtime_config();
13213        let spec = windowed_crash_spec("budget-journal-failure");
13214        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13215        assert!(matches!(
13216            on_child_exit(
13217                &spec,
13218                runtime.restart_policy,
13219                &supervisor.registry,
13220                &snapshot,
13221                &runtime.terminal_ring,
13222                &runtime.spawn_events,
13223                &runtime.child_roster,
13224                crash_exit_report(1),
13225                true,
13226            )
13227            .await,
13228            NextAction::Stop { .. }
13229        ));
13230        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13231        let history = runtime
13232            .terminal_ring
13233            .lock()
13234            .unwrap()
13235            .durable_history(&spec.module_id);
13236        assert!(history.journal_write_failures > 0);
13237        assert_eq!(history.entries.len(), 1);
13238        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13239    }
13240
13241    #[test]
13242    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13243        let supervisor = Supervisor::new_for_test(
13244            Arc::new(Registry::default()),
13245            RestartPolicy::new(0, Duration::ZERO),
13246        );
13247        let runtime = supervisor.runtime_config();
13248        let spec = windowed_crash_spec("budget-recording-panic");
13249        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13250        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13251        // Exhausting the event sequence makes emit_exited panic before the
13252        // terminal write, exercising publication on the recording unwind.
13253        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13254        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13255            tokio::runtime::Builder::new_current_thread()
13256                .enable_all()
13257                .build()
13258                .unwrap()
13259                .block_on(on_child_exit(
13260                    &spec,
13261                    runtime.restart_policy,
13262                    &supervisor.registry,
13263                    &snapshot,
13264                    &runtime.terminal_ring,
13265                    &runtime.spawn_events,
13266                    &runtime.child_roster,
13267                    crash_exit_report(1),
13268                    true,
13269                ))
13270        }));
13271        let panic = result.err().expect("recording must still unwind");
13272        assert_eq!(
13273            panic.downcast_ref::<String>().map(String::as_str),
13274            Some("spawn event sequence exhausted")
13275        );
13276        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13277    }
13278
13279    /// A stop the supervisor itself requests still stops a protocol-none
13280    /// module, even though the child answers the SIGTERM with exit 0.
13281    #[cfg(unix)]
13282    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13283    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13284        for disable in [false, true] {
13285            let label = if disable {
13286                "none-requested-disable"
13287            } else {
13288                "none-requested-stop"
13289            };
13290            let dir = cortexkit_test_support::ScratchDir::new(label);
13291            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13292            let supervisor = Supervisor::new_for_test(
13293                Arc::new(Registry::default()),
13294                RestartPolicy::new(3, Duration::ZERO),
13295            );
13296            let module = supervisor.spawn(spec).unwrap();
13297            wait_for_file(&ready).await;
13298
13299            if disable {
13300                module.set_enabled(false).await.unwrap();
13301            } else {
13302                module.stop().await.unwrap();
13303            }
13304            assert!(
13305                marker.exists(),
13306                "{label}: the child must have left through its SIGTERM handler with exit 0"
13307            );
13308
13309            // Long enough for a zero-backoff respawn to have happened if the
13310            // exit had been treated as a crash.
13311            sleep(Duration::from_millis(500)).await;
13312            let status = module.status().unwrap();
13313            let expected = if disable {
13314                ModuleState::Disabled
13315            } else {
13316                ModuleState::Stopped
13317            };
13318            assert_eq!(status.state, expected, "{label}");
13319            assert_eq!(
13320                status.spawn_generation, 1,
13321                "{label}: respawned after a requested stop"
13322            );
13323            let history = module.terminal_history();
13324            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13325            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13326            assert_ne!(
13327                history.entries[0].disposition,
13328                TerminalDisposition::Restarting,
13329                "{label}"
13330            );
13331        }
13332    }
13333
13334    /// A subc-wire module that exits 0 on its own is still a stop: the
13335    /// protocol-none rule must not reach it.
13336    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13337    async fn subc_wire_clean_exit_is_still_a_stop() {
13338        let supervisor = Supervisor::new_for_test(
13339            Arc::new(Registry::default()),
13340            RestartPolicy::new(3, Duration::ZERO),
13341        );
13342        let module = supervisor
13343            .spawn(ModuleSpec {
13344                module_id: "wire-clean-exit".to_string(),
13345                program: fake_aft_stub_path(),
13346                args: Vec::new(),
13347                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13348                reserved: false,
13349                reserved_prefixes: Vec::new(),
13350                protocol: ModuleProtocol::Subc,
13351                overlap: Default::default(),
13352            })
13353            .unwrap();
13354
13355        let deadline = Instant::now() + Duration::from_secs(10);
13356        while module.terminal_history().entries.is_empty() {
13357            assert!(Instant::now() < deadline, "module never exited");
13358            sleep(Duration::from_millis(10)).await;
13359        }
13360        // Long enough for a zero-backoff respawn to have happened.
13361        sleep(Duration::from_millis(500)).await;
13362        let status = module.status().unwrap();
13363        assert_eq!(status.state, ModuleState::Stopped);
13364        assert_eq!(status.spawn_generation, 1);
13365        let history = module.terminal_history();
13366        assert_eq!(history.entries.len(), 1, "{history:?}");
13367        assert_eq!(history.entries[0].exit_code, Some(0));
13368        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13369    }
13370
13371    #[cfg(unix)]
13372    #[tokio::test]
13373    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13374        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13375        let record = dir.join("live-children.json");
13376        let supervisor = Supervisor::new_for_test(
13377            Arc::new(Registry::default()),
13378            RestartPolicy::new(0, Duration::ZERO),
13379        );
13380        let mut runtime = supervisor.runtime_config();
13381        runtime.child_roster.record_to(record.clone());
13382        let gate = Arc::new(super::ReloadExitRecordGate::default());
13383        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13384        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13385        let spec = ModuleSpec {
13386            module_id: "reload-exit-roster".into(),
13387            program: fake_aft_stub_path(),
13388            args: Vec::new(),
13389            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13390            reserved: false,
13391            reserved_prefixes: Vec::new(),
13392            protocol: ModuleProtocol::Subc,
13393            overlap: Default::default(),
13394        };
13395        let mut child = None;
13396        let reload = super::finish_reload_child(
13397            &spec,
13398            &runtime,
13399            &supervisor.registry,
13400            &supervisor.process_liveness,
13401            &snapshot,
13402            &mut child,
13403        );
13404        tokio::pin!(reload);
13405        tokio::select! {
13406            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13407            _ = gate.reached.notified() => {}
13408        }
13409        assert!(runtime
13410            .terminal_ring
13411            .lock()
13412            .unwrap()
13413            .snapshot()
13414            .entries
13415            .is_empty());
13416        assert_eq!(
13417            crate::live_children::read_record(&record).unwrap().len(),
13418            1,
13419            "shutdown must still wait for the reaped child until its terminal record exists"
13420        );
13421        runtime.child_roster.close();
13422        gate.resume.notify_one();
13423        assert!(reload.await.is_err());
13424        assert!(crate::live_children::read_record(&record)
13425            .unwrap()
13426            .is_empty());
13427        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13428        assert_eq!(history.entries.len(), 1);
13429        assert_eq!(
13430            history.entries[0].disposition,
13431            TerminalDisposition::DaemonShutdown
13432        );
13433    }
13434
13435    /// Each restart-producing arm has its own state transition. Keeping their
13436    /// lifetime count assertions adjacent prevents a later new arm from silently
13437    /// spending budget without recording the historical restart.
13438    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13439    async fn every_restart_increment_path_advances_lifetime_count() {
13440        let supervisor = Supervisor::new_for_test(
13441            Arc::new(Registry::default()),
13442            RestartPolicy::new(1, Duration::ZERO),
13443        );
13444        let runtime = supervisor.runtime_config();
13445        let spec = ModuleSpec {
13446            module_id: "lifetime-increment-path".to_string(),
13447            program: PathBuf::from("/unused/lifetime-increment-path"),
13448            args: Vec::new(),
13449            env: Vec::new(),
13450            reserved: false,
13451            reserved_prefixes: Vec::new(),
13452            protocol: ModuleProtocol::Subc,
13453            overlap: Default::default(),
13454        };
13455
13456        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13457        assert!(matches!(
13458            on_child_exit(
13459                &spec,
13460                runtime.restart_policy,
13461                &supervisor.registry,
13462                &crash_snapshot,
13463                &runtime.terminal_ring,
13464                &runtime.spawn_events,
13465                &runtime.child_roster,
13466                ExitReport {
13467                    kind: ExitKind::Crash,
13468                    code: Some(1),
13469                    signal: None,
13470                    at_ms: 1,
13471                },
13472                true,
13473            )
13474            .await,
13475            NextAction::Restart { schedule: _ }
13476        ));
13477        let (crash_restarts, crash_lifetime) = {
13478            let state = lock_snapshot(&crash_snapshot).unwrap();
13479            (state.crash_restarts.len(), state.lifetime_restarts)
13480        };
13481        assert_eq!(crash_restarts, 1);
13482        assert_eq!(crash_lifetime, 1);
13483
13484        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13485        let mut health_child = None;
13486        assert!(matches!(
13487            health_restart_child(
13488                &spec,
13489                &runtime,
13490                &supervisor.registry,
13491                &supervisor.process_liveness,
13492                &health_snapshot,
13493                &mut health_child,
13494                SupervisorHealthStatus::Failing,
13495                None,
13496                2,
13497            )
13498            .await,
13499            Ok(())
13500        ));
13501        assert!(health_child.is_none());
13502        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13503        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13504        let (health_restarts, health_lifetime) = {
13505            let state = lock_snapshot(&health_snapshot).unwrap();
13506            (state.crash_restarts.len(), state.lifetime_restarts)
13507        };
13508        assert_eq!(health_restarts, 1);
13509        assert_eq!(health_lifetime, 1);
13510
13511        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13512        let mut reload_child = None;
13513        assert!(matches!(
13514            handle_reload_spawn_failure(
13515                &spec,
13516                &runtime,
13517                &supervisor.process_liveness,
13518                &reload_snapshot,
13519                &mut reload_child,
13520                "forced reload spawn failure".to_string(),
13521            )
13522            .await,
13523            Err(SuperviseError::ReloadFailed { .. })
13524        ));
13525        let (reload_restarts, reload_lifetime) = {
13526            let state = lock_snapshot(&reload_snapshot).unwrap();
13527            (state.crash_restarts.len(), state.lifetime_restarts)
13528        };
13529        assert_eq!(reload_restarts, 1);
13530        assert_eq!(reload_lifetime, 1);
13531    }
13532
13533    #[tokio::test]
13534    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13535        let supervisor = Supervisor::new_for_test(
13536            Arc::new(Registry::default()),
13537            RestartPolicy::new(3, Duration::ZERO),
13538        );
13539        let runtime = supervisor.runtime_config();
13540        let spec = ModuleSpec {
13541            module_id: "deliberately-severed".to_string(),
13542            program: PathBuf::from("/unused/deliberately-severed"),
13543            args: Vec::new(),
13544            env: Vec::new(),
13545            reserved: false,
13546            reserved_prefixes: Vec::new(),
13547            protocol: ModuleProtocol::Subc,
13548            overlap: Default::default(),
13549        };
13550        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13551        let process = ProcessIdentity {
13552            pid: 41,
13553            start_time: 101,
13554        };
13555        record_deliberate_severance(&snapshot, process).unwrap();
13556        let exit_report = apply_deliberate_severance_marker(
13557            &snapshot,
13558            Some(process),
13559            ExitReport {
13560                kind: ExitKind::Crash,
13561                code: Some(1),
13562                signal: None,
13563                at_ms: 1,
13564            },
13565        );
13566        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13567
13568        assert!(matches!(
13569            on_child_exit(
13570                &spec,
13571                runtime.restart_policy,
13572                &supervisor.registry,
13573                &snapshot,
13574                &runtime.terminal_ring,
13575                &runtime.spawn_events,
13576                &runtime.child_roster,
13577                exit_report,
13578                true,
13579            )
13580            .await,
13581            NextAction::Restart { schedule: _ }
13582        ));
13583        let state = lock_snapshot(&snapshot).unwrap();
13584        assert_eq!(state.lifetime_restarts, 1);
13585        assert_eq!(state.crash_restarts.len(), 0);
13586    }
13587
13588    #[tokio::test]
13589    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13590        let supervisor = Supervisor::new_for_test(
13591            Arc::new(Registry::default()),
13592            RestartPolicy::new(3, Duration::ZERO),
13593        );
13594        let runtime = supervisor.runtime_config();
13595        let spec = ModuleSpec {
13596            module_id: "genuine-crash".to_string(),
13597            program: PathBuf::from("/unused/genuine-crash"),
13598            args: Vec::new(),
13599            env: Vec::new(),
13600            reserved: false,
13601            reserved_prefixes: Vec::new(),
13602            protocol: ModuleProtocol::Subc,
13603            overlap: Default::default(),
13604        };
13605        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13606
13607        assert!(matches!(
13608            on_child_exit(
13609                &spec,
13610                runtime.restart_policy,
13611                &supervisor.registry,
13612                &snapshot,
13613                &runtime.terminal_ring,
13614                &runtime.spawn_events,
13615                &runtime.child_roster,
13616                ExitReport {
13617                    kind: ExitKind::Crash,
13618                    code: Some(1),
13619                    signal: None,
13620                    at_ms: 1,
13621                },
13622                true,
13623            )
13624            .await,
13625            NextAction::Restart { schedule: _ }
13626        ));
13627        let state = lock_snapshot(&snapshot).unwrap();
13628        assert_eq!(state.lifetime_restarts, 1);
13629        assert_eq!(state.crash_restarts.len(), 1);
13630    }
13631
13632    fn crash_exit_report(at_ms: u64) -> ExitReport {
13633        ExitReport {
13634            kind: ExitKind::Crash,
13635            code: Some(1),
13636            signal: None,
13637            at_ms,
13638        }
13639    }
13640
13641    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13642        ModuleSpec {
13643            module_id: module_id.to_string(),
13644            program: PathBuf::from("/unused").join(module_id),
13645            args: Vec::new(),
13646            env: Vec::new(),
13647            reserved: false,
13648            reserved_prefixes: Vec::new(),
13649            protocol: ModuleProtocol::Subc,
13650            overlap: Default::default(),
13651        }
13652    }
13653
13654    /// A real crash loop still stops. Three crashes with nothing aging out spend
13655    /// a budget of two and the third respawn is refused, and both surfaces an
13656    /// operator has -- the log line and the retained terminal record -- name the
13657    /// window rather than only the cap, because `max_restarts=2` alone is what
13658    /// this budget used to mean.
13659    #[tokio::test]
13660    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13661        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13662        let supervisor = Supervisor::new_for_test(
13663            Arc::new(Registry::default()),
13664            RestartPolicy::new(2, Duration::ZERO),
13665        );
13666        let runtime = supervisor.runtime_config();
13667        let spec = windowed_crash_spec("crash-loop-in-window");
13668        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13669
13670        for attempt in 1..=2 {
13671            assert!(
13672                matches!(
13673                    on_child_exit(
13674                        &spec,
13675                        runtime.restart_policy,
13676                        &supervisor.registry,
13677                        &snapshot,
13678                        &runtime.terminal_ring,
13679                        &runtime.spawn_events,
13680                        &runtime.child_roster,
13681                        crash_exit_report(attempt),
13682                        true,
13683                    )
13684                    .await,
13685                    NextAction::Restart { schedule: _ }
13686                ),
13687                "crash {attempt} is inside the budget and must respawn"
13688            );
13689        }
13690
13691        assert!(matches!(
13692            on_child_exit(
13693                &spec,
13694                runtime.restart_policy,
13695                &supervisor.registry,
13696                &snapshot,
13697                &runtime.terminal_ring,
13698                &runtime.spawn_events,
13699                &runtime.child_roster,
13700                crash_exit_report(3),
13701                true,
13702            )
13703            .await,
13704            NextAction::Stop { .. }
13705        ));
13706
13707        {
13708            let state = lock_snapshot(&snapshot).unwrap();
13709            assert_eq!(state.state, ModuleState::Failed);
13710            assert_eq!(state.crash_restarts.len(), 2);
13711            assert_eq!(state.lifetime_restarts, 2);
13712        }
13713
13714        let history = runtime
13715            .terminal_ring
13716            .lock()
13717            .expect("terminal ring is not poisoned")
13718            .snapshot();
13719        let last = history
13720            .entries
13721            .last()
13722            .expect("the refused crash is retained");
13723        assert_eq!(last.disposition, TerminalDisposition::Failed);
13724        assert_eq!(
13725            last.disposition_detail.as_deref(),
13726            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13727        );
13728
13729        let captured = crate::router::test_log::captured_logs(&logs);
13730        assert!(
13731            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13732            "the stop must be logged with its window: {captured}"
13733        );
13734    }
13735
13736    /// The rate, stated as a test: three crashes where the first has aged past
13737    /// the window are two crashes as far as the budget is concerned, so the
13738    /// third respawn is allowed and the ring holds only the two recent ones.
13739    ///
13740    /// This is the case a lifetime counter got wrong -- and the case the daemon
13741    /// now hits routinely, since a module exits non-zero every time its
13742    /// connection to the daemon drops.
13743    #[tokio::test]
13744    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13745        let supervisor = Supervisor::new_for_test(
13746            Arc::new(Registry::default()),
13747            RestartPolicy::new(2, Duration::ZERO),
13748        );
13749        let runtime = supervisor.runtime_config();
13750        let spec = windowed_crash_spec("crash-across-windows");
13751        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13752
13753        for attempt in 1..=2 {
13754            assert!(matches!(
13755                on_child_exit(
13756                    &spec,
13757                    runtime.restart_policy,
13758                    &supervisor.registry,
13759                    &snapshot,
13760                    &runtime.terminal_ring,
13761                    &runtime.spawn_events,
13762                    &runtime.child_roster,
13763                    crash_exit_report(attempt),
13764                    true,
13765                )
13766                .await,
13767                NextAction::Restart { schedule: _ }
13768            ));
13769        }
13770
13771        // The oldest crash moves out of the window; nothing else about the
13772        // module changes.
13773        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13774            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13775        })
13776        .unwrap();
13777
13778        assert!(
13779            matches!(
13780                on_child_exit(
13781                    &spec,
13782                    runtime.restart_policy,
13783                    &supervisor.registry,
13784                    &snapshot,
13785                    &runtime.terminal_ring,
13786                    &runtime.spawn_events,
13787                    &runtime.child_roster,
13788                    crash_exit_report(3),
13789                    true,
13790                )
13791                .await,
13792                NextAction::Restart { schedule: _ }
13793            ),
13794            "a crash older than the window must not hold a budget slot"
13795        );
13796
13797        let state = lock_snapshot(&snapshot).unwrap();
13798        assert_eq!(state.state, ModuleState::Restarting);
13799        assert_eq!(
13800            state.crash_restarts.len(),
13801            2,
13802            "the aged instant is dropped and the new one takes its place"
13803        );
13804        assert_eq!(
13805            state.lifetime_restarts, 3,
13806            "the ledger counts every restart, including the ones the window forgot"
13807        );
13808    }
13809
13810    /// An operator restart hands the budget back whole, and the ledger keeps
13811    /// counting. Those are different questions -- "how close is this module to
13812    /// being stopped" and "how many times has it been replaced" -- and the
13813    /// operator action answers only the first.
13814    #[tokio::test]
13815    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13816        let supervisor = Supervisor::new_for_test(
13817            Arc::new(Registry::default()),
13818            RestartPolicy::new(2, Duration::ZERO),
13819        );
13820        let runtime = supervisor.runtime_config();
13821        let spec = windowed_crash_spec("operator-cleared-budget");
13822        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13823
13824        for attempt in 1..=2 {
13825            assert!(matches!(
13826                on_child_exit(
13827                    &spec,
13828                    runtime.restart_policy,
13829                    &supervisor.registry,
13830                    &snapshot,
13831                    &runtime.terminal_ring,
13832                    &runtime.spawn_events,
13833                    &runtime.child_roster,
13834                    crash_exit_report(attempt),
13835                    true,
13836                )
13837                .await,
13838                NextAction::Restart { schedule: _ }
13839            ));
13840        }
13841
13842        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13843        {
13844            let state = lock_snapshot(&snapshot).unwrap();
13845            assert!(
13846                state.crash_restarts.is_empty(),
13847                "an operator restart returns the full budget"
13848            );
13849            assert_eq!(
13850                state.lifetime_restarts, 2,
13851                "clearing the budget must not unmake the crashes"
13852            );
13853        }
13854
13855        assert!(
13856            matches!(
13857                on_child_exit(
13858                    &spec,
13859                    runtime.restart_policy,
13860                    &supervisor.registry,
13861                    &snapshot,
13862                    &runtime.terminal_ring,
13863                    &runtime.spawn_events,
13864                    &runtime.child_roster,
13865                    crash_exit_report(3),
13866                    true,
13867                )
13868                .await,
13869                NextAction::Restart { schedule: _ }
13870            ),
13871            "the cleared budget must be spendable again"
13872        );
13873        let state = lock_snapshot(&snapshot).unwrap();
13874        assert_eq!(state.crash_restarts.len(), 1);
13875        assert_eq!(state.lifetime_restarts, 3);
13876    }
13877
13878    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13879    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13880        let severed = ProcessIdentity {
13881            pid: 41,
13882            start_time: 101,
13883        };
13884        let successor = ProcessIdentity {
13885            pid: 41,
13886            start_time: 202,
13887        };
13888        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13889        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13890            state.pid = Some(successor.pid);
13891            state.process_start_time = Some(successor.start_time);
13892        })
13893        .unwrap();
13894        assert!(!module.record_deliberate_severance(severed).unwrap());
13895
13896        let exit_report = apply_deliberate_severance_marker(
13897            &module.inner.snapshot,
13898            Some(successor),
13899            ExitReport {
13900                kind: ExitKind::Crash,
13901                code: Some(1),
13902                signal: None,
13903                at_ms: 1,
13904            },
13905        );
13906
13907        assert_eq!(exit_report.kind, ExitKind::Crash);
13908    }
13909
13910    #[tokio::test]
13911    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13912        let registry = Registry::default();
13913        let supervisor = Supervisor::new_for_test(
13914            Arc::new(Registry::default()),
13915            RestartPolicy::new(3, Duration::ZERO),
13916        );
13917        let runtime = supervisor.runtime_config();
13918        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13919        let spec = ModuleSpec {
13920            module_id: "drain-deliberate-severance".to_string(),
13921            program: fake_aft_stub_path(),
13922            args: Vec::new(),
13923            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13924            reserved: false,
13925            reserved_prefixes: Vec::new(),
13926            protocol: ModuleProtocol::Subc,
13927            overlap: Default::default(),
13928        };
13929        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13930        let process = ProcessIdentity {
13931            pid: 41,
13932            start_time: 101,
13933        };
13934        child.process_identity = Some(process);
13935        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13936            state.pid = Some(process.pid);
13937            state.process_start_time = Some(process.start_time);
13938        })
13939        .unwrap();
13940        record_deliberate_severance(&snapshot, process).unwrap();
13941
13942        drain_child_to_state(
13943            &spec.module_id,
13944            spec.protocol,
13945            // The child exits on its own; no signal may change the exit this
13946            // test classifies.
13947            StopNotice::SentOverConnection,
13948            &registry,
13949            None,
13950            &snapshot,
13951            &runtime.terminal_ring,
13952            &runtime.spawn_events,
13953            child,
13954            Duration::from_secs(1),
13955            ModuleState::Stopped,
13956            Some(false),
13957            &super::FailureContext::operator(),
13958        )
13959        .await
13960        .unwrap();
13961
13962        let state = lock_snapshot(&snapshot).unwrap();
13963        assert_eq!(
13964            state.last_exit.as_ref().map(|exit| exit.kind),
13965            Some(ExitKind::DeliberateSeverance)
13966        );
13967        assert_eq!(state.lifetime_restarts, 1);
13968        assert_eq!(state.crash_restarts.len(), 0);
13969        drop(state);
13970        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13971        assert_eq!(
13972            history.entries[0].exit_kind,
13973            subc_control::TerminalExitKind::DeliberateSeverance
13974        );
13975    }
13976
13977    #[tokio::test]
13978    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13979        let registry = Registry::default();
13980        let supervisor = Supervisor::new_for_test(
13981            Arc::new(Registry::default()),
13982            RestartPolicy::new(3, Duration::ZERO),
13983        );
13984        let runtime = supervisor.runtime_config();
13985        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13986        let spec = ModuleSpec {
13987            module_id: "ordinary-drain".to_string(),
13988            program: fake_aft_stub_path(),
13989            args: Vec::new(),
13990            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13991            reserved: false,
13992            reserved_prefixes: Vec::new(),
13993            protocol: ModuleProtocol::Subc,
13994            overlap: Default::default(),
13995        };
13996        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13997
13998        drain_child_to_state(
13999            &spec.module_id,
14000            spec.protocol,
14001            // The child exits on its own; no signal may change the exit this
14002            // test classifies.
14003            StopNotice::SentOverConnection,
14004            &registry,
14005            None,
14006            &snapshot,
14007            &runtime.terminal_ring,
14008            &runtime.spawn_events,
14009            child,
14010            Duration::from_secs(1),
14011            ModuleState::Stopped,
14012            Some(false),
14013            &super::FailureContext::operator(),
14014        )
14015        .await
14016        .unwrap();
14017
14018        let state = lock_snapshot(&snapshot).unwrap();
14019        assert_eq!(
14020            state.last_exit.as_ref().map(|exit| exit.kind),
14021            Some(ExitKind::Crash)
14022        );
14023        assert_eq!(state.lifetime_restarts, 0);
14024        assert_eq!(state.crash_restarts.len(), 0);
14025    }
14026
14027    #[test]
14028    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14029        // The server's generic fatal-routing branch only knows that the
14030        // connection failed; it does not know that the daemon deliberately
14031        // initiated a process-killing severance. Keep this seam explicit so a
14032        // future connection error path cannot silently reintroduce the stale
14033        // exemption that mislabels a later genuine crash.
14034        assert!(!include_str!("server.rs")
14035            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14036    }
14037
14038    /// The `route.closed` `drained` value must be the quiescence wait's own
14039    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
14040    /// measurement at all and `false` is the one honest constant. This is the exact
14041    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
14042    /// on every return path, including the one that used to return early via `?`
14043    /// with `route.closing` already sent and no `route.closed` ever following.
14044    #[test]
14045    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14046        assert!(drained_after_quiescence_wait(&Ok(true)));
14047        assert!(!drained_after_quiescence_wait(&Ok(false)));
14048        assert!(!drained_after_quiescence_wait(&Err(
14049            SuperviseError::StatePoisoned { module_id: None }
14050        )));
14051    }
14052
14053    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
14054    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
14055    /// already reaped out-of-band) still leaves a terminal record rather than none
14056    /// at all. Triggering the real `wait()` I/O error from an integration test would
14057    /// need a genuine already-reaped-child race, which is OS-specific and not
14058    /// something this suite attempts elsewhere; this test instead verifies the
14059    /// record produced for that arm end-to-end through the real `TerminalRing`, and
14060    /// the call site itself is verified by inspection to sit in that exact arm.
14061    #[test]
14062    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14063        let ring = Arc::new(Mutex::new(TerminalRing::new(
14064            TerminalRingConfig::default(),
14065            0,
14066        )));
14067        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14068
14069        let snapshot = ring.lock().unwrap().snapshot();
14070        assert_eq!(snapshot.entries.len(), 1);
14071        let entry = &snapshot.entries[0];
14072        assert_eq!(entry.exit_code, None);
14073        assert_eq!(entry.exit_signal, None);
14074        assert_eq!(entry.disposition, TerminalDisposition::Failed);
14075    }
14076
14077    #[test]
14078    fn wait_error_exit_path_preserves_spawn_event_density() {
14079        let feed = super::SpawnEventFeed::default();
14080        feed.configure_incarnation("wait-error-density".to_string());
14081        feed.emit_spawned("wait-error", 41, 1);
14082        let ring = Arc::new(Mutex::new(TerminalRing::new(
14083            TerminalRingConfig::default(),
14084            0,
14085        )));
14086
14087        record_wait_error_terminal("wait-error", &ring, &feed);
14088        feed.emit_spawned("after-wait-error", 42, 2);
14089
14090        let state = feed.0.lock().unwrap();
14091        let sequences = state
14092            .events
14093            .iter()
14094            .map(|event| event.cursor.seq)
14095            .collect::<Vec<_>>();
14096        assert_eq!(sequences, vec![1, 2, 3]);
14097        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14098        assert_eq!(state.events[1].exit_code, None);
14099        assert_eq!(state.events[1].exit_signal, None);
14100    }
14101
14102    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
14103    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
14104    /// not a clean exit it never actually observed.
14105    #[test]
14106    fn wait_error_exit_report_is_classified_as_a_crash() {
14107        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14108    }
14109}
14110
14111#[cfg(all(test, unix))]
14112mod restart_ordering_tests {
14113    use super::*;
14114    use std::io::Write;
14115    use tokio::io::AsyncReadExt;
14116
14117    #[test]
14118    #[ignore = "child process fixture"]
14119    fn serving_child() {
14120        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14121            return;
14122        };
14123        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14124        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14125        for stream in listener.incoming() {
14126            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14127        }
14128    }
14129
14130    async fn restart_before_publication(requests: usize) {
14131        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14132        let address_file = home.path().join("address");
14133        let spec = ModuleSpec {
14134            module_id: "restart-serving-child".into(),
14135            program: std::env::current_exe().unwrap(),
14136            args: vec![
14137                "--exact".into(),
14138                "supervise::restart_ordering_tests::serving_child".into(),
14139                "--ignored".into(),
14140                "--nocapture".into(),
14141            ],
14142            env: vec![(
14143                "RESTART_TEST_ADDRESS_FILE".into(),
14144                address_file.display().to_string(),
14145            )],
14146            reserved: false,
14147            reserved_prefixes: vec![],
14148            protocol: ModuleProtocol::None,
14149            overlap: Default::default(),
14150        };
14151        let mut supervisor =
14152            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14153        let (spawned_tx, spawned_rx) = oneshot::channel();
14154        let spawned_tx = Mutex::new(Some(spawned_tx));
14155        let (release_tx, release_rx) = std::sync::mpsc::channel();
14156        let release_rx = Mutex::new(release_rx);
14157        // Hold publication, not the child: it must answer a real request before
14158        // the restart enters the queue, even though status still has generation 0.
14159        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14160            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14161            release_rx
14162                .lock()
14163                .unwrap()
14164                .recv_timeout(Duration::from_secs(20))
14165                .unwrap();
14166        })));
14167        let registered = supervisor
14168            .register_configured_with_health(
14169                spec.clone(),
14170                true,
14171                HealthConfig::default(),
14172                None,
14173                RestartPolicy::new(3, Duration::ZERO),
14174                DEFAULT_REGISTER_TIMEOUT,
14175            )
14176            .unwrap();
14177        let module = registered.module.clone();
14178        let startup = tokio::task::spawn_blocking(move || {
14179            supervisor.start_registered_configured(registered, None, true)
14180        });
14181        spawned_rx.await.unwrap();
14182        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14183            let address = loop {
14184                if let Ok(address) = std::fs::read_to_string(&address_file) {
14185                    if !address.is_empty() {
14186                        break address;
14187                    }
14188                }
14189                tokio::time::sleep(Duration::from_millis(1)).await;
14190            };
14191            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14192            let mut reply = String::new();
14193            stream.read_to_string(&mut reply).await.unwrap();
14194            reply.trim().parse::<u32>().unwrap()
14195        })
14196        .await
14197        .expect("child must serve before generation publication");
14198        assert_eq!(
14199            lock_snapshot(&module.inner.snapshot)
14200                .unwrap()
14201                .spawn_generation,
14202            0
14203        );
14204        let mut replies = Vec::new();
14205        for _ in 0..requests {
14206            let mut restart = Box::pin(module.restart(Some(0)));
14207            // Poll the public API through queue admission, leaving only its ACK
14208            // pending. The monitor cannot consume the command until released.
14209            std::future::poll_fn(|cx| {
14210                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14211                std::task::Poll::Ready(())
14212            })
14213            .await;
14214            replies.push(restart);
14215        }
14216        release_tx.send(()).unwrap();
14217        startup.await.unwrap().unwrap();
14218        for reply in replies {
14219            reply.await.unwrap();
14220        }
14221        // Restart schedules its replacement after completing the handler. Wait
14222        // for that separate actor turn before placing the command barrier.
14223        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14224            loop {
14225                if lock_snapshot(&module.inner.snapshot)
14226                    .unwrap()
14227                    .spawn_generation
14228                    > 1
14229                {
14230                    break;
14231                }
14232                tokio::time::sleep(Duration::from_millis(1)).await;
14233            }
14234        })
14235        .await;
14236        // Commands are handled serially. The update-spec reply proves every
14237        // restart handler has finished before we inspect the snapshot; restart's
14238        // own ACK only acknowledges initiation, not replacement completion.
14239        module.update_spec_for_test(spec).await.unwrap();
14240        let (generation, pid) = {
14241            let state = lock_snapshot(&module.inner.snapshot).unwrap();
14242            (state.spawn_generation, state.pid)
14243        };
14244        module.stop().await.unwrap();
14245        replacement.expect("restart must publish a replacement for the already-serving child");
14246        assert_ne!(
14247            pid,
14248            Some(old_pid),
14249            "a serving child predating the restart must be replaced"
14250        );
14251        assert_eq!(generation, 2, "queued restarts must share one replacement");
14252    }
14253
14254    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14255    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14256        restart_before_publication(1).await;
14257    }
14258
14259    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14260    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14261        restart_before_publication(2).await;
14262    }
14263
14264    #[tokio::test]
14265    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14266        let supervisor =
14267            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14268        let registered = supervisor
14269            .register_configured_with_health(
14270                ModuleSpec {
14271                    module_id: "queued-before-replacement".into(),
14272                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
14273                    // macOS a module whose running image is not the configured
14274                    // program is refused once its exec is confirmed (see
14275                    // `record_write_stall_tests::sleeping_spec`).
14276                    program: "/bin/sleep".into(),
14277                    args: vec!["600".into()],
14278                    env: vec![],
14279                    reserved: false,
14280                    reserved_prefixes: vec![],
14281                    protocol: ModuleProtocol::None,
14282                    overlap: Default::default(),
14283                },
14284                true,
14285                HealthConfig::default(),
14286                None,
14287                RestartPolicy::default(),
14288                DEFAULT_REGISTER_TIMEOUT,
14289            )
14290            .unwrap();
14291        let RegisteredConfiguredModule {
14292            mut spec,
14293            mut runtime,
14294            module,
14295            mut commands,
14296        } = registered;
14297        let snapshot = &module.inner.snapshot;
14298        let mut restart = Box::pin(module.restart(Some(0)));
14299        std::future::poll_fn(|cx| {
14300            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14301            std::task::Poll::Ready(())
14302        })
14303        .await;
14304        // Drive the actor explicitly so a replacement is definitely spawned
14305        // after receipt, but before this queued command is dequeued.
14306        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14307        let replacement_pid = child.as_ref().unwrap().id();
14308        let command = commands.recv().await.unwrap();
14309        assert!(
14310            handle_supervisor_command(
14311                command,
14312                &mut spec,
14313                &mut runtime,
14314                &supervisor.registry,
14315                &supervisor.process_liveness,
14316                snapshot,
14317                &mut child,
14318                &mut commands,
14319                &mut VecDeque::new(),
14320            )
14321            .await
14322        );
14323        restart.await.unwrap();
14324        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14325        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14326        if let Some(mut child) = child {
14327            child.start_kill().unwrap();
14328            child.wait().await.unwrap();
14329        }
14330        assert_eq!(
14331            kept_pid, replacement_pid,
14332            "a newer replacement must survive"
14333        );
14334        assert!(
14335            !pending,
14336            "a satisfied restart must not schedule another spawn"
14337        );
14338    }
14339}
14340
14341#[cfg(all(test, unix))]
14342mod health_event_tests {
14343    use super::*;
14344    use cortexkit_test_support::ScratchDir;
14345    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14346
14347    struct Actor {
14348        module: SupervisedModule,
14349        registry: Arc<Registry>,
14350        forwarding: Arc<ForwardingTable>,
14351        spec: ModuleSpec,
14352        health: HealthConfig,
14353        rx: mpsc::Receiver<crate::router::OutboundFrame>,
14354        roster: ChildRoster,
14355        _home: ScratchDir,
14356    }
14357
14358    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14359        // OS exec and socket readiness are real I/O. Stay runnable while waiting
14360        // for an observed condition, rather than letting paused time auto-advance
14361        // deadlines or assuming a fixed number of yields finishes the work.
14362        let deadline = std::time::Instant::now() + Duration::from_secs(10);
14363        let tick = Instant::now();
14364        loop {
14365            if let Some(value) = observe() {
14366                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14367                return value;
14368            }
14369            assert!(
14370                std::time::Instant::now() < deadline,
14371                "timed out waiting for {reason}"
14372            );
14373            tokio::task::yield_now().await;
14374        }
14375    }
14376
14377    impl Actor {
14378        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14379            Self::start_with_policy(
14380                protocol,
14381                cadence,
14382                DEFAULT_REGISTER_TIMEOUT,
14383                RestartPolicy::new(3, Duration::ZERO),
14384            )
14385            .await
14386        }
14387
14388        async fn start_with_policy(
14389            protocol: ModuleProtocol,
14390            cadence: Duration,
14391            register_timeout: Duration,
14392            policy: RestartPolicy,
14393        ) -> Self {
14394            let home = ScratchDir::new("health-events");
14395            let registry = Arc::new(Registry::default());
14396            let forwarding = Arc::new(ForwardingTable::default());
14397            let health = HealthConfig {
14398                cadence,
14399                deadline: Duration::from_secs(3600),
14400                ..HealthConfig::default()
14401            };
14402            let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14403                .with_forwarding(forwarding.clone())
14404                .with_health_config(health.clone());
14405            let spec = ModuleSpec {
14406                module_id: "event-health-child".into(),
14407                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
14408                // the privacy trampoline refuses a module whose running image
14409                // is not the configured program, and `start` below waits for
14410                // the process to be confirmed (`reported_pid`). The shell form
14411                // passed only while the image check ran before the shell had
14412                // replaced itself with `sleep` (see
14413                // `record_write_stall_tests::sleeping_spec`).
14414                program: PathBuf::from("/bin/sleep"),
14415                // The process stays alive but has no real bus peer. Registrations
14416                // below exercise the same registry writes as accepted HELLOs.
14417                args: vec!["600".into()],
14418                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14419                    .into_iter()
14420                    .map(|key| (key.into(), home.path().display().to_string()))
14421                    .collect(),
14422                reserved: false,
14423                reserved_prefixes: vec![],
14424                protocol,
14425                overlap: Default::default(),
14426            };
14427            let module = supervisor
14428                .supervise_configured_requested(
14429                    spec.clone(),
14430                    true,
14431                    health.clone(),
14432                    Some(0),
14433                    policy,
14434                    register_timeout,
14435                    None,
14436                )
14437                .unwrap();
14438            let (_, rx) = mpsc::channel(8);
14439            let actor = Self {
14440                module,
14441                registry,
14442                forwarding,
14443                spec,
14444                health,
14445                rx,
14446                roster: supervisor.child_roster.clone(),
14447                _home: home,
14448            };
14449            actor
14450                .wait_for_select(|checkpoint| checkpoint.generation > 0)
14451                .await;
14452            actor
14453        }
14454
14455        async fn expire_registration_wait(&self, remaining: Duration) {
14456            tokio::time::advance(remaining).await;
14457            wait_for("registration timeout TERM grace", || {
14458                lock_snapshot(&self.module.inner.snapshot)
14459                    .unwrap()
14460                    .actor_select
14461                    .is_none()
14462                    .then_some(())
14463            })
14464            .await;
14465            tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14466        }
14467
14468        fn turns(&self) -> u64 {
14469            lock_snapshot(&self.module.inner.snapshot)
14470                .unwrap()
14471                .actor_turns
14472        }
14473
14474        async fn wait_for_select(
14475            &self,
14476            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14477        ) -> ActorSelectCheckpoint {
14478            wait_for("actor select checkpoint", || {
14479                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14480                state.actor_select.filter(|checkpoint| {
14481                    checkpoint.generation == state.spawn_generation
14482                        && state.state == ModuleState::Running
14483                        && state.process_alive
14484                        && state.reported_pid().is_some()
14485                        && expected(checkpoint)
14486                })
14487            })
14488            .await
14489        }
14490
14491        fn hello(&mut self, connection: u64, health: bool) {
14492            let manifest =
14493                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14494                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14495                    .build();
14496            let connection = ConnectionId::new(connection);
14497            let (tx, rx) = mpsc::channel(8);
14498            self.rx = rx;
14499            self.forwarding
14500                .register_module_connection(
14501                    connection,
14502                    self.spec.module_id.clone(),
14503                    subc_protocol::PROTOCOL_VERSION,
14504                    Concurrency::ModuleManaged,
14505                    FrameSink::new(tx),
14506                )
14507                .unwrap();
14508            self.registry
14509                .register_with_control_ops(
14510                    manifest,
14511                    subc_protocol::PROTOCOL_VERSION,
14512                    connection,
14513                    if health {
14514                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14515                    } else {
14516                        vec![]
14517                    },
14518                )
14519                .unwrap();
14520        }
14521
14522        async fn probe(&mut self) -> crate::router::OutboundFrame {
14523            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14524                Ok(frame) => Some(frame),
14525                Err(mpsc::error::TryRecvError::Empty) => None,
14526                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14527            })
14528            .await;
14529            let body: Value = serde_json::from_slice(&frame.body).unwrap();
14530            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14531            frame
14532        }
14533
14534        fn no_probe(&mut self) {
14535            assert!(matches!(
14536                self.rx.try_recv(),
14537                Err(mpsc::error::TryRecvError::Empty)
14538            ));
14539        }
14540
14541        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14542            self.forwarding
14543                .complete_module_control_rpc(
14544                    ConnectionId::new(connection),
14545                    frame.header.corr,
14546                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14547                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14548                        status: HealthStatus::Ok,
14549                        detail: None,
14550                        metrics: None,
14551                    }),
14552                )
14553                .unwrap();
14554        }
14555    }
14556
14557    #[tokio::test(start_paused = true)]
14558    async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14559        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14560        let initial = actor.module.status().unwrap();
14561        tokio::time::advance(Duration::from_secs(119)).await;
14562        tokio::task::yield_now().await;
14563        assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14564        assert!(actor.module.terminal_history().entries.is_empty());
14565        actor.expire_registration_wait(Duration::from_secs(1)).await;
14566        actor
14567            .wait_for_select(|at| at.generation > initial.spawn_generation)
14568            .await;
14569        let status = actor.module.status().unwrap();
14570        assert_ne!(status.pid, initial.pid);
14571        assert!(!cortexkit_test_support::process_alive(
14572            initial.pid.unwrap() as i32
14573        ));
14574        assert_eq!(status.restart_count, 1);
14575        assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14576        let entries = actor.module.terminal_history().entries;
14577        assert_eq!(entries.len(), 1);
14578        assert_eq!(
14579            entries[0].disposition_detail.as_deref(),
14580            Some("registration_timeout")
14581        );
14582        actor.module.stop().await.unwrap();
14583    }
14584
14585    #[tokio::test(start_paused = true)]
14586    async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14587        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14588        let pid = actor.module.status().unwrap().pid;
14589        tokio::time::advance(Duration::from_secs(119)).await;
14590        actor.hello(1, false);
14591        let registered = actor
14592            .wait_for_select(|at| at.registered_connection.is_some())
14593            .await;
14594        assert_eq!(
14595            registered.wake_after, None,
14596            "HELLO must cancel the deadline"
14597        );
14598        tokio::time::advance(Duration::from_secs(300)).await;
14599        tokio::task::yield_now().await;
14600        assert_eq!(actor.module.status().unwrap().pid, pid);
14601        assert!(actor.module.terminal_history().entries.is_empty());
14602        // A disconnect after the first HELLO must not re-arm a startup timer.
14603        actor
14604            .registry
14605            .deregister_connection(ConnectionId::new(1))
14606            .unwrap();
14607        actor
14608            .wait_for_select(|at| at.registered_connection.is_none())
14609            .await;
14610        tokio::time::advance(Duration::from_secs(300)).await;
14611        tokio::task::yield_now().await;
14612        assert_eq!(actor.module.status().unwrap().pid, pid);
14613        assert!(actor.module.terminal_history().entries.is_empty());
14614        actor.module.stop().await.unwrap();
14615
14616        let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14617        let pid = transient.module.status().unwrap().pid;
14618        tokio::time::advance(Duration::from_secs(119)).await;
14619        let turns = transient.turns();
14620        // Both writes happen before the supervisor can refresh the registry.
14621        // An accepted HELLO must remain evidence after its connection is gone.
14622        transient.hello(1, false);
14623        transient
14624            .registry
14625            .deregister_connection(ConnectionId::new(1))
14626            .unwrap();
14627        let parked = transient.wait_for_select(|at| at.turn > turns).await;
14628        assert_eq!(
14629            parked.wake_after, None,
14630            "transient HELLO must cancel the deadline"
14631        );
14632        tokio::time::advance(Duration::from_secs(300)).await;
14633        tokio::task::yield_now().await;
14634        assert_eq!(transient.module.status().unwrap().pid, pid);
14635        assert!(transient.module.terminal_history().entries.is_empty());
14636        transient.module.stop().await.unwrap();
14637    }
14638
14639    #[tokio::test(start_paused = true)]
14640    async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14641        let actor = Actor::start_with_policy(
14642            ModuleProtocol::Subc,
14643            Duration::from_secs(30),
14644            Duration::from_secs(10),
14645            RestartPolicy::new(1, Duration::ZERO),
14646        )
14647        .await;
14648        let generation = actor.module.status().unwrap().spawn_generation;
14649        actor
14650            .expire_registration_wait(Duration::from_secs(10))
14651            .await;
14652        actor.wait_for_select(|at| at.generation > generation).await;
14653        actor
14654            .expire_registration_wait(Duration::from_secs(10))
14655            .await;
14656        let retry = wait_for("failed retry schedule", || {
14657            let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14658            (state.state == ModuleState::Failed)
14659                .then_some(state.failed_retry.pending)
14660                .flatten()
14661        })
14662        .await;
14663        assert_eq!(retry.delay, Duration::from_secs(600));
14664        assert_eq!(retry.attempt, 1);
14665        let entries = actor.module.terminal_history().entries;
14666        assert_eq!(entries.len(), 2);
14667        assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14668        assert!(entries[1]
14669            .disposition_detail
14670            .as_deref()
14671            .unwrap()
14672            .contains("registration_timeout"));
14673        tokio::time::advance(Duration::from_secs(599)).await;
14674        tokio::task::yield_now().await;
14675        assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14676        tokio::time::advance(Duration::from_secs(1)).await;
14677        actor
14678            .wait_for_select(|at| at.generation > generation + 1)
14679            .await;
14680        // The slow retry is another ordinary spawn, with its own HELLO budget.
14681        actor
14682            .expire_registration_wait(Duration::from_secs(10))
14683            .await;
14684        wait_for("slow retry registration timeout", || {
14685            (actor.module.terminal_history().entries.len() == 3).then_some(())
14686        })
14687        .await;
14688        actor.module.stop().await.unwrap();
14689    }
14690
14691    #[tokio::test(start_paused = true)]
14692    async fn registration_timeout_honours_ten_second_override() {
14693        let actor = Actor::start_with_policy(
14694            ModuleProtocol::Subc,
14695            Duration::from_secs(30),
14696            Duration::from_secs(10),
14697            RestartPolicy::new(3, Duration::ZERO),
14698        )
14699        .await;
14700        let generation = actor.module.status().unwrap().spawn_generation;
14701        tokio::time::advance(Duration::from_secs(9)).await;
14702        tokio::task::yield_now().await;
14703        assert!(actor.module.terminal_history().entries.is_empty());
14704        actor.expire_registration_wait(Duration::from_secs(1)).await;
14705        actor.wait_for_select(|at| at.generation > generation).await;
14706        assert_eq!(actor.module.terminal_history().entries.len(), 1);
14707        actor.module.stop().await.unwrap();
14708    }
14709
14710    #[tokio::test]
14711    async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14712        use std::os::unix::process::{CommandExt, ExitStatusExt};
14713
14714        struct Helper(std::process::Child);
14715        impl Drop for Helper {
14716            fn drop(&mut self) {
14717                // This handle still owns the helper until wait returns. Never
14718                // clean up its old process group after the leader is reaped.
14719                if matches!(self.0.try_wait(), Ok(None)) {
14720                    let _ = self.0.kill();
14721                    let _ = self.0.wait();
14722                }
14723            }
14724        }
14725
14726        let home = ScratchDir::new("registration-timeout-helper");
14727        let supervisor = Supervisor::new_for_test(
14728            Arc::new(Registry::default()),
14729            RestartPolicy::new(0, Duration::ZERO),
14730        );
14731        let spec = ModuleSpec {
14732            module_id: "registration-timeout-group".into(),
14733            program: "/bin/sleep".into(),
14734            args: vec!["120".into()],
14735            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14736                .into_iter()
14737                .map(|key| (key.into(), home.path().display().to_string()))
14738                .collect(),
14739            reserved: false,
14740            reserved_prefixes: vec![],
14741            protocol: ModuleProtocol::Subc,
14742            overlap: Default::default(),
14743        };
14744        let module = supervisor
14745            .supervise_configured_requested(
14746                spec,
14747                true,
14748                HealthConfig::default(),
14749                None,
14750                RestartPolicy::new(0, Duration::ZERO),
14751                Duration::from_secs(10),
14752                None,
14753            )
14754            .unwrap();
14755        let leader = timeout(Duration::from_secs(5), async {
14756            loop {
14757                if let Some(pid) = module.status().unwrap().pid {
14758                    break pid;
14759                }
14760                sleep(Duration::from_millis(5)).await;
14761            }
14762        })
14763        .await
14764        .expect("supervised group leader must be reported before the deadline");
14765        assert!(leader > 1);
14766        let ready = home.join("helper.ready");
14767        // Own the helper separately so even a broken group kill can be cleaned
14768        // up by its own unreaped child handle, without signalling a stale pgid.
14769        let mut helper = Helper(
14770            std::process::Command::new("/bin/sh")
14771                .args([
14772                    "-c",
14773                    "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14774                    "helper",
14775                ])
14776                .arg(&ready)
14777                .process_group(i32::try_from(leader).unwrap())
14778                .stdout(std::process::Stdio::null())
14779                .stderr(std::process::Stdio::null())
14780                .spawn()
14781                .unwrap(),
14782        );
14783        timeout(Duration::from_secs(5), async {
14784            while !ready.exists() {
14785                assert!(helper.0.try_wait().unwrap().is_none());
14786                sleep(Duration::from_millis(5)).await;
14787            }
14788        })
14789        .await
14790        .expect("helper must install TERM ignore before the registration deadline");
14791        let helper_pid =
14792            rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14793        rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14794        sleep(Duration::from_millis(20)).await;
14795        assert!(
14796            helper.0.try_wait().unwrap().is_none(),
14797            "helper must ignore TERM"
14798        );
14799
14800        timeout(Duration::from_secs(20), async {
14801            while module.state().unwrap() != ModuleState::Failed {
14802                sleep(Duration::from_millis(5)).await;
14803            }
14804        })
14805        .await
14806        .expect("registration timeout must stop and reap the group leader");
14807        let helper_exit = timeout(Duration::from_secs(3), async {
14808            loop {
14809                if let Some(status) = helper.0.try_wait().unwrap() {
14810                    break status;
14811                }
14812                sleep(Duration::from_millis(5)).await;
14813            }
14814        })
14815        .await
14816        .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14817        assert_eq!(helper_exit.signal(), Some(9));
14818        assert!(!cortexkit_test_support::process_alive(
14819            i32::try_from(leader).unwrap()
14820        ));
14821        assert_eq!(
14822            module.status().unwrap().last_exit.unwrap().kind,
14823            ExitKind::Crash
14824        );
14825        assert!(module.terminal_history().entries[0]
14826            .disposition_detail
14827            .as_deref()
14828            .unwrap()
14829            .contains("registration_timeout"));
14830        module.stop().await.unwrap();
14831    }
14832
14833    #[tokio::test(start_paused = true)]
14834    async fn registration_timeout_does_not_count_an_earlier_exit() {
14835        let actor = Actor::start_with_policy(
14836            ModuleProtocol::Subc,
14837            Duration::from_secs(30),
14838            DEFAULT_REGISTER_TIMEOUT,
14839            RestartPolicy::new(0, Duration::ZERO),
14840        )
14841        .await;
14842        let pid = actor.module.status().unwrap().pid.unwrap();
14843        rustix::process::kill_process(
14844            rustix::process::Pid::from_raw(pid as i32).unwrap(),
14845            rustix::process::Signal::TERM,
14846        )
14847        .unwrap();
14848        wait_for("early exit", || {
14849            (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14850        })
14851        .await;
14852        tokio::time::advance(Duration::from_secs(121)).await;
14853        tokio::task::yield_now().await;
14854        let entries = actor.module.terminal_history().entries;
14855        assert_eq!(entries.len(), 1);
14856        assert!(!entries[0]
14857            .disposition_detail
14858            .as_deref()
14859            .unwrap()
14860            .contains("registration_timeout"));
14861        assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14862        actor.module.stop().await.unwrap();
14863    }
14864
14865    #[tokio::test(start_paused = true)]
14866    async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14867        let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14868        let pid = actor.module.status().unwrap().pid;
14869        tokio::time::advance(Duration::from_secs(119)).await;
14870        actor.roster.close();
14871        let turns = actor.turns();
14872        tokio::time::advance(Duration::from_secs(300)).await;
14873        let parked = actor.wait_for_select(|at| at.turn > turns).await;
14874        assert_eq!(parked.wake_after, None);
14875        assert_eq!(actor.module.status().unwrap().pid, pid);
14876        assert!(actor.module.terminal_history().entries.is_empty());
14877        actor.module.stop().await.unwrap();
14878        let status = actor.module.status().unwrap();
14879        assert_eq!(status.restart_count, 0);
14880        assert_eq!(
14881            actor.module.terminal_history().entries[0].disposition,
14882            TerminalDisposition::DaemonShutdown
14883        );
14884    }
14885
14886    #[tokio::test(start_paused = true)]
14887    async fn no_health_children_park_for_sixty_seconds() {
14888        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14889        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14890        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14891        unadvertised.hello(1, false);
14892        // Include any one-off registration wake in the budget; receiving a
14893        // no-health HELLO must not turn a parked child into a polling child.
14894        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
14895        // a single 60 s jump would only observe one expired timer.
14896        for _ in 0..6000 {
14897            tokio::time::advance(Duration::from_millis(10)).await;
14898            tokio::task::yield_now().await;
14899        }
14900        for actor in [&none, &unregistered, &unadvertised] {
14901            assert!(
14902                actor.turns() <= 5,
14903                "no-health actor woke {} times",
14904                actor.turns()
14905            );
14906            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14907        }
14908    }
14909
14910    #[tokio::test(start_paused = true)]
14911    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14912        // With a zero cadence the first probe is due at once, so this measures
14913        // only how fast the registration notification starts health probing,
14914        // not the normal 30-33 s wait before a first probe (checked below).
14915        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14916        let tick = Instant::now();
14917        actor.hello(1, true);
14918        actor.probe().await;
14919        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14920    }
14921
14922    #[tokio::test(start_paused = true)]
14923    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14924        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14925        let hello_tick = Instant::now();
14926        actor.hello(1, true);
14927        let armed = actor
14928            .wait_for_select(|checkpoint| {
14929                checkpoint.registered_connection == Some(ConnectionId::new(1))
14930                    && checkpoint.next_probe_at.is_some()
14931            })
14932            .await;
14933        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14934        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14935        let first_deadline = hello_tick + delay;
14936        assert_eq!(armed.next_probe_at, Some(first_deadline));
14937        tokio::time::advance(Duration::from_secs(29)).await;
14938        let turns = actor.turns();
14939        actor
14940            .registry
14941            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14942            .unwrap();
14943        let updated = actor
14944            .wait_for_select(|checkpoint| checkpoint.turn > turns)
14945            .await;
14946        assert_eq!(
14947            updated.next_probe_at,
14948            Some(first_deadline),
14949            "catalog update must not reset cadence"
14950        );
14951        actor.no_probe();
14952        tokio::time::advance(first_deadline - Instant::now()).await;
14953        let frame = actor.probe().await;
14954        actor.answer(1, frame);
14955        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14956        let next_deadline = Instant::now() + next;
14957        let rearmed = actor
14958            .wait_for_select(|checkpoint| {
14959                checkpoint
14960                    .next_probe_at
14961                    .is_some_and(|deadline| deadline > first_deadline)
14962            })
14963            .await;
14964        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14965        tokio::time::advance(next - Duration::from_nanos(1)).await;
14966        actor.no_probe();
14967        tokio::time::advance(Duration::from_nanos(1)).await;
14968        actor.probe().await;
14969    }
14970
14971    #[tokio::test(start_paused = true)]
14972    async fn disconnect_disarms_and_reconnect_rearms_health() {
14973        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14974        actor.hello(1, true);
14975        actor
14976            .wait_for_select(|checkpoint| {
14977                checkpoint.registered_connection == Some(ConnectionId::new(1))
14978                    && checkpoint.next_probe_at.is_some()
14979            })
14980            .await;
14981        let turns = actor.turns();
14982        let tick = Instant::now();
14983        actor
14984            .registry
14985            .deregister_connection(ConnectionId::new(1))
14986            .unwrap();
14987        let disarmed = actor
14988            .wait_for_select(|checkpoint| {
14989                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14990            })
14991            .await;
14992        assert_eq!(disarmed.next_probe_at, None);
14993        assert_eq!(disarmed.wake_after, None);
14994        assert_eq!(
14995            Instant::now(),
14996            tick,
14997            "disconnect must disarm in the notification tick"
14998        );
14999        tokio::time::advance(Duration::from_secs(60)).await;
15000        actor.no_probe();
15001        let reconnect_tick = Instant::now();
15002        actor.hello(2, true);
15003        let rearmed = actor
15004            .wait_for_select(|checkpoint| {
15005                checkpoint.registered_connection == Some(ConnectionId::new(2))
15006                    && checkpoint.next_probe_at.is_some()
15007            })
15008            .await;
15009        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15010        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15011        tokio::time::advance(delay).await;
15012        actor.probe().await;
15013    }
15014
15015    #[tokio::test(start_paused = true)]
15016    async fn reregistration_without_health_disarms_the_old_schedule() {
15017        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15018        actor.hello(1, true);
15019        actor
15020            .wait_for_select(|checkpoint| {
15021                checkpoint.registered_connection == Some(ConnectionId::new(1))
15022                    && checkpoint.next_probe_at.is_some()
15023            })
15024            .await;
15025        actor
15026            .registry
15027            .deregister_connection(ConnectionId::new(1))
15028            .unwrap();
15029        actor.hello(2, false);
15030        let disarmed = actor
15031            .wait_for_select(|checkpoint| {
15032                checkpoint.registered_connection == Some(ConnectionId::new(2))
15033            })
15034            .await;
15035        assert_eq!(disarmed.next_probe_at, None);
15036        assert_eq!(disarmed.wake_after, None);
15037        tokio::time::advance(Duration::from_secs(60)).await;
15038        actor.no_probe();
15039    }
15040
15041    #[tokio::test(start_paused = true)]
15042    async fn restart_waits_for_its_own_hello_before_rearming_health() {
15043        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15044        actor.hello(1, true);
15045        let armed = actor
15046            .wait_for_select(|checkpoint| {
15047                checkpoint.registered_connection == Some(ConnectionId::new(1))
15048                    && checkpoint.next_probe_at.is_some()
15049            })
15050            .await;
15051        // Simulate the old process's connection closing before the restart
15052        // tears the process down. The replacement process has started but has
15053        // not sent its HELLO, so it is not registered.
15054        actor
15055            .registry
15056            .deregister_connection(ConnectionId::new(1))
15057            .unwrap();
15058        actor.module.restart(Some(0)).await.unwrap();
15059        let parked = actor
15060            .wait_for_select(|checkpoint| {
15061                checkpoint.generation > armed.generation
15062                    && checkpoint.registered_connection.is_none()
15063                    && checkpoint.next_probe_at.is_none()
15064            })
15065            .await;
15066        assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15067        actor.rx.close();
15068        // Drain the stop notice the restart queued for the old process's connection,
15069        // so it isn't mistaken for traffic to the replacement.
15070        while actor.rx.try_recv().is_ok() {}
15071        let turns = actor.turns();
15072        tokio::time::advance(Duration::from_secs(60)).await;
15073        let after = actor.turns();
15074        eprintln!(
15075            "restart park turns: before={turns} after={after} delta={}",
15076            after - turns
15077        );
15078        assert_eq!(after, turns, "replacement must park before HELLO");
15079        let hello_tick = Instant::now();
15080        actor.hello(2, true);
15081        let rearmed = actor
15082            .wait_for_select(|checkpoint| {
15083                checkpoint.registered_connection == Some(ConnectionId::new(2))
15084                    && checkpoint.next_probe_at.is_some()
15085            })
15086            .await;
15087        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15088        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15089        tokio::time::advance(delay).await;
15090        actor.probe().await;
15091    }
15092
15093    #[tokio::test(start_paused = true)]
15094    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15095        use tokio::{
15096            io::{AsyncReadExt, AsyncWriteExt},
15097            net::TcpListener,
15098        };
15099        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15100        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15101        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15102        let (started_tx, mut started_rx) = mpsc::channel(8);
15103        let peer = tokio::spawn(async move {
15104            loop {
15105                let (mut socket, _) = listener.accept().await.unwrap();
15106                let mut request = [0; 1024];
15107                assert!(socket.read(&mut request).await.unwrap() > 0);
15108                started_tx.send(Instant::now()).await.unwrap();
15109                socket
15110                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15111                    .await
15112                    .unwrap();
15113            }
15114        });
15115        actor
15116            .module
15117            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15118            .await
15119            .unwrap();
15120        let first_tick = Instant::now();
15121        let first = actor
15122            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15123            .await;
15124        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15125        assert!(started_rx.try_recv().is_err());
15126
15127        // Restart just before the old deadline: reusing it would give the new
15128        // process only one second to start, rather than a full cadence.
15129        tokio::time::advance(
15130            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15131        )
15132        .await;
15133        actor.module.restart(Some(0)).await.unwrap();
15134        let spawn_tick = Instant::now();
15135        let replacement = actor
15136            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15137            .await;
15138        let deadline = replacement.next_probe_at.unwrap();
15139        assert!(
15140            deadline >= spawn_tick + actor.health.cadence,
15141            "a replacement must not inherit its predecessor's probe deadline"
15142        );
15143        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15144        assert!(
15145            started_rx.try_recv().is_err(),
15146            "no HTTP probe during startup grace"
15147        );
15148        tokio::time::advance(deadline - Instant::now()).await;
15149        let landed = wait_for("first replacement HTTP probe", || {
15150            started_rx.try_recv().ok()
15151        })
15152        .await;
15153        assert!(landed >= spawn_tick + actor.health.cadence);
15154        wait_for("healthy replacement", || {
15155            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15156                .then_some(())
15157        })
15158        .await;
15159        peer.abort();
15160    }
15161
15162    #[tokio::test(start_paused = true)]
15163    async fn rescan_adds_and_removes_http_health_without_polling() {
15164        use tokio::{
15165            io::{AsyncReadExt, AsyncWriteExt},
15166            net::TcpListener,
15167        };
15168        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15169        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15170        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15171        // One millisecond separates successive probes without a polling delay.
15172        actor.health.cadence = Duration::from_millis(1);
15173        let (started_tx, mut started_rx) = mpsc::channel(8);
15174        let peer = tokio::spawn(async move {
15175            loop {
15176                let (mut socket, _) = listener.accept().await.unwrap();
15177                let mut request = [0; 1024];
15178                assert!(socket.read(&mut request).await.unwrap() > 0);
15179                started_tx.send(()).await.unwrap();
15180                socket
15181                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15182                    .await
15183                    .unwrap();
15184            }
15185        });
15186        let tick = Instant::now();
15187        let turns = actor.turns();
15188        actor
15189            .module
15190            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15191            .await
15192            .unwrap();
15193        let armed = actor
15194            .wait_for_select(|checkpoint| {
15195                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15196            })
15197            .await;
15198        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15199        assert_eq!(Instant::now(), tick);
15200        tokio::time::advance(Duration::from_millis(1)).await;
15201        let mut started = false;
15202        wait_for("successful HTTP probe", || {
15203            started |= started_rx.try_recv().is_ok();
15204            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15205                .then_some(())
15206        })
15207        .await;
15208        assert!(started, "added HTTP check must start at its first deadline");
15209        assert_eq!(
15210            actor.module.status().unwrap().health.status,
15211            SupervisorHealthStatus::Ok
15212        );
15213        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15214        actor.health.http = None;
15215        let turns = actor.turns();
15216        actor
15217            .module
15218            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15219            .await
15220            .unwrap();
15221        let parked = actor
15222            .wait_for_select(|checkpoint| {
15223                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15224            })
15225            .await;
15226        assert_eq!(parked.wake_after, None);
15227        let turns = actor.turns();
15228        tokio::time::advance(Duration::from_secs(60)).await;
15229        assert_eq!(
15230            actor.turns(),
15231            turns,
15232            "removed HTTP check must park the actor"
15233        );
15234        assert!(
15235            started_rx.try_recv().is_err(),
15236            "removed HTTP check must stay disarmed"
15237        );
15238        assert_eq!(
15239            actor.module.status().unwrap().health,
15240            ModuleHealthStatus::default()
15241        );
15242        peer.abort();
15243    }
15244}
15245
15246#[cfg(test)]
15247mod drain_busy_reprobe_tests {
15248    //! The drain's busy-gauge wait, driven against an in-process module
15249    //! connection whose health answers the test scripts. Time is paused, so
15250    //! every elapsed duration below is exact virtual time.
15251    use super::*;
15252    use std::collections::BTreeMap;
15253    use std::sync::Mutex as StdMutex;
15254    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15255
15256    const MODULE_ID: &str = "drain-reprobe-module";
15257    const CONNECTION: u64 = 7;
15258    const GAUGE: &str = "runs_in_flight";
15259    /// The production default for both the health cadence and the drain
15260    /// ceiling, which is the pairing that used to cost a full ceiling.
15261    const CADENCE: Duration = Duration::from_secs(30);
15262    const CEILING: Duration = Duration::from_secs(30);
15263
15264    /// How the scripted module answers each `health.check`.
15265    #[derive(Clone, Copy)]
15266    enum Answer {
15267        /// Report the gauge busy until this instant, quiet from then on, after
15268        /// taking `latency` to answer.
15269        BusyUntil { until: Instant, latency: Duration },
15270        /// Answer at once, with metrics that leave the declared gauge out.
15271        OmitsGauge,
15272        /// Never answer.
15273        Silent,
15274    }
15275
15276    struct Drain {
15277        forwarding: Arc<ForwardingTable>,
15278        runtime: SupervisorRuntimeConfig,
15279        endpoint: crate::ModuleEndpointId,
15280        /// When each probe reached the module.
15281        probes: Arc<StdMutex<Vec<Instant>>>,
15282        responder: tokio::task::JoinHandle<()>,
15283    }
15284
15285    impl Drain {
15286        fn start(answer: Answer) -> Self {
15287            let registry = Arc::new(Registry::default());
15288            let forwarding = Arc::new(ForwardingTable::default());
15289            let health = HealthConfig {
15290                cadence: CADENCE,
15291                ..HealthConfig::default()
15292            };
15293            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15294                .with_forwarding(forwarding.clone())
15295                .with_health_config(health)
15296                .runtime_config();
15297            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15298            let endpoint = forwarding
15299                .register_module_connection(
15300                    ConnectionId::new(CONNECTION),
15301                    MODULE_ID.to_string(),
15302                    subc_protocol::PROTOCOL_VERSION,
15303                    Concurrency::ModuleManaged,
15304                    FrameSink::new(tx),
15305                )
15306                .unwrap();
15307            let probes = Arc::new(StdMutex::new(Vec::new()));
15308            let responder = tokio::spawn({
15309                let forwarding = forwarding.clone();
15310                let probes = probes.clone();
15311                async move {
15312                    while let Some(frame) = rx.recv().await {
15313                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
15314                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15315                        probes.lock().unwrap().push(Instant::now());
15316                        let metrics = match answer {
15317                            Answer::BusyUntil { until, latency } => {
15318                                sleep(latency).await;
15319                                let busy = u64::from(Instant::now() < until);
15320                                serde_json::json!({ GAUGE: busy })
15321                            }
15322                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15323                            Answer::Silent => continue,
15324                        };
15325                        // A probe abandoned at its deadline has no RPC left to
15326                        // complete, so a late answer may be refused.
15327                        let _ = forwarding.complete_module_control_rpc(
15328                            ConnectionId::new(CONNECTION),
15329                            frame.header.corr,
15330                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15331                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15332                                status: HealthStatus::Ok,
15333                                detail: None,
15334                                metrics: Some(metrics),
15335                            }),
15336                        );
15337                    }
15338                }
15339            });
15340            Self {
15341                forwarding,
15342                runtime,
15343                endpoint,
15344                probes,
15345                responder,
15346            }
15347        }
15348
15349        /// Run one by-id drain wait with `ceiling`, returning its verdict and
15350        /// how long it took.
15351        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15352            let (drained, elapsed, _) = self.run_traced(ceiling).await;
15353            (drained, elapsed)
15354        }
15355
15356        /// [`Self::run`], also returning what the drain's probes read.
15357        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15358            let started = Instant::now();
15359            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15360            let drained = wait_for_forwarding_quiescence(
15361                &self.forwarding,
15362                MODULE_ID,
15363                &self.runtime,
15364                self.endpoint,
15365                started + ceiling,
15366                &[GAUGE.to_string()],
15367                DrainScope::Active,
15368                &mut trace,
15369            )
15370            .await
15371            .unwrap();
15372            (drained, started.elapsed(), trace)
15373        }
15374
15375        fn probes(&self) -> Vec<Instant> {
15376            self.probes.lock().unwrap().clone()
15377        }
15378    }
15379
15380    impl Drop for Drain {
15381        fn drop(&mut self) {
15382            self.responder.abort();
15383        }
15384    }
15385
15386    #[tokio::test(start_paused = true)]
15387    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15388        let drain = Drain::start(Answer::BusyUntil {
15389            until: Instant::now() + Duration::from_secs(1),
15390            latency: Duration::ZERO,
15391        });
15392        let (drained, elapsed) = drain.run(CEILING).await;
15393        assert!(
15394            drained,
15395            "gauges that read zero during the drain must let it finish drained, \
15396             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15397        );
15398        assert!(
15399            elapsed < Duration::from_secs(3),
15400            "drain took {elapsed:?} after the gauges went quiet at 1s"
15401        );
15402        let probes = drain.probes();
15403        assert!(
15404            probes.len() >= 2,
15405            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15406        );
15407    }
15408
15409    /// The same shape in real time on a multi-threaded runtime: the first
15410    /// probe is answered busy at once, within a millisecond of the drain
15411    /// starting, and later probes read idle. The drain must keep probing
15412    /// after that first instant answer and end soon after the gauge falls.
15413    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15414    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15415        let drain = Drain::start(Answer::BusyUntil {
15416            until: Instant::now() + Duration::from_millis(600),
15417            latency: Duration::ZERO,
15418        });
15419        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15420        assert!(drained, "the gauge fell at 600ms: {trace}");
15421        assert!(
15422            elapsed < Duration::from_secs(3),
15423            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15424        );
15425        let probes = drain.probes();
15426        assert!(
15427            probes.len() >= 2 && trace.probes as usize == probes.len(),
15428            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15429        );
15430    }
15431
15432    #[tokio::test(start_paused = true)]
15433    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15434        let started = Instant::now();
15435        let drain = Drain::start(Answer::BusyUntil {
15436            until: started + Duration::from_secs(3600),
15437            latency: Duration::ZERO,
15438        });
15439        let (drained, elapsed) = drain.run(CEILING).await;
15440        assert!(!drained, "a module busy throughout must not drain");
15441        assert!(
15442            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15443            "a busy module is held until the ceiling and no longer: {elapsed:?}"
15444        );
15445        // Re-probing is paced by the interval, not a busy loop: at most one
15446        // probe per interval across the ceiling, plus the first at time zero.
15447        let probes = drain.probes();
15448        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15449        assert!(
15450            probes.len() as u128 <= max_probes,
15451            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15452            probes.len()
15453        );
15454        for pair in probes.windows(2) {
15455            assert!(
15456                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15457                "consecutive probes {:?} apart",
15458                pair[1] - pair[0]
15459            );
15460        }
15461    }
15462
15463    #[tokio::test(start_paused = true)]
15464    async fn a_slow_probe_is_never_overlapped_by_the_next() {
15465        // Each answer takes longer than the re-probe interval; the next probe
15466        // must wait for the previous answer and then the interval.
15467        let latency = Duration::from_secs(1);
15468        let drain = Drain::start(Answer::BusyUntil {
15469            until: Instant::now() + Duration::from_secs(3600),
15470            latency,
15471        });
15472        let (drained, _) = drain.run(Duration::from_secs(5)).await;
15473        assert!(!drained);
15474        let probes = drain.probes();
15475        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15476        for pair in probes.windows(2) {
15477            assert!(
15478                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15479                "a probe started {:?} after the previous one, before it could finish",
15480                pair[1] - pair[0]
15481            );
15482        }
15483    }
15484
15485    #[tokio::test(start_paused = true)]
15486    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15487        let drain = Drain::start(Answer::Silent);
15488        let ceiling = Duration::from_secs(2);
15489        let (drained, elapsed) = drain.run(ceiling).await;
15490        assert!(!drained, "an unanswered probe counts as busy");
15491        assert!(
15492            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15493            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15494        );
15495        assert_eq!(
15496            drain.probes().len(),
15497            1,
15498            "the unanswered probe holds the only slot until the ceiling"
15499        );
15500    }
15501
15502    /// Every event logged while it is installed, as `message` plus fields.
15503    #[derive(Clone, Default)]
15504    struct LogCapture {
15505        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15506    }
15507
15508    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15509        fn on_event(
15510            &self,
15511            event: &tracing::Event<'_>,
15512            _context: tracing_subscriber::layer::Context<'_, S>,
15513        ) {
15514            struct Fields(BTreeMap<String, String>);
15515            impl tracing::field::Visit for Fields {
15516                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15517                    self.0
15518                        .insert(field.name().to_string(), format!("{value:?}"));
15519                }
15520            }
15521            let mut fields = Fields(BTreeMap::new());
15522            event.record(&mut fields);
15523            self.events.lock().unwrap().push(fields.0);
15524        }
15525    }
15526
15527    impl LogCapture {
15528        fn install(&self) -> tracing::subscriber::DefaultGuard {
15529            use tracing_subscriber::layer::SubscriberExt;
15530            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15531        }
15532
15533        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15534            self.events
15535                .lock()
15536                .unwrap()
15537                .iter()
15538                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15539                .cloned()
15540                .collect()
15541        }
15542    }
15543
15544    /// A module that declares a busy gauge but leaves it out of its health
15545    /// answers holds a drain to its ceiling. The timeout line must say so by
15546    /// name, and the drain must warn about the missing gauge once, not on
15547    /// every probe.
15548    #[tokio::test(start_paused = true)]
15549    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15550        let capture = LogCapture::default();
15551        let _guard = capture.install();
15552        let drain = Drain::start(Answer::OmitsGauge);
15553        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15554        assert!(!drained, "a gauge never reported cannot read quiet");
15555        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15556
15557        let missing = capture.with_message("lacks a declared busy gauge");
15558        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15559        assert_eq!(
15560            missing[0].get("missing_gauges").map(String::as_str),
15561            Some(GAUGE)
15562        );
15563
15564        warn_drain_timed_out(
15565            MODULE_ID,
15566            Duration::from_secs(2),
15567            RouteCloseReason::Restart,
15568            &Default::default(),
15569            &trace,
15570        );
15571        let timeouts = capture.with_message("route drain timed out");
15572        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15573        let gauges = &timeouts[0]["busy_gauges"];
15574        assert!(
15575            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15576            "the timeout line must name the gauge and why it read busy: {gauges}"
15577        );
15578        assert!(
15579            gauges.ends_with("; no value ever read"),
15580            "no value was ever read for the gauge: {gauges}"
15581        );
15582        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15583    }
15584
15585    /// The trace names a busy value and when it was read. For a probe that
15586    /// never got an answer, it names the probe's error kind and says that no
15587    /// value was ever read.
15588    #[tokio::test(start_paused = true)]
15589    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15590        let busy = Drain::start(Answer::BusyUntil {
15591            until: Instant::now() + Duration::from_secs(3600),
15592            latency: Duration::ZERO,
15593        });
15594        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15595        let rendered = trace.to_string();
15596        assert!(
15597            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15598                && rendered.contains("; last value non_zero(1) @"),
15599            "{rendered}"
15600        );
15601
15602        let silent = Drain::start(Answer::Silent);
15603        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15604        // The probe is capped at the 1 s ceiling, and its reading is stamped
15605        // when it gave up.
15606        assert_eq!(
15607            trace.to_string(),
15608            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15609        );
15610        assert_eq!(trace.probes, 1);
15611    }
15612
15613    /// A gauge value that is not a count reads busy and is shown as sent.
15614    #[test]
15615    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15616        let gauges = [GAUGE.to_string()];
15617        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15618        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15619        assert_eq!(
15620            busy_gauge_observation(&verdicts),
15621            BusyGaugeObservation::Busy
15622        );
15623        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15624        assert_eq!(
15625            busy_gauge_observation(&zero),
15626            BusyGaugeObservation::Quiescent
15627        );
15628        let none = gauge_verdicts(None, &gauges);
15629        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15630    }
15631}
15632
15633#[cfg(test)]
15634mod health_evidence_tests {
15635    use super::{HealthProbeError, HealthProbeEvidence};
15636    use std::collections::HashSet;
15637
15638    /// The evidential asymmetry, asserted rather than described.
15639    ///
15640    /// Exactly ONE observation is proof a module cannot serve, and the one that
15641    /// fires under CPU starvation is not it. Before the split, all fifteen
15642    /// construction sites collapsed into a single String, so a timeout carried the
15643    /// same weight as a dead lane -- which is how a healthy module was restarted
15644    /// three times in one day.
15645    #[test]
15646    fn only_a_dead_lane_is_proof_of_death() {
15647        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15648        // Three non-proof classes, each for a different reason: silence is
15649        // consistent with health, a bad answer proves the module ALIVE, and a
15650        // daemon-side fault never reached the module at all.
15651        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15652        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15653        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15654    }
15655
15656    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
15657    ///
15658    /// A shared label renders two different observations identically in the line an
15659    /// operator reads after an unexplained restart -- the exact confusion this
15660    /// change removes.
15661    #[test]
15662    fn every_evidence_class_has_a_distinct_label() {
15663        let labels = [
15664            HealthProbeError::lane_dead("").label(),
15665            HealthProbeError::no_answer("").label(),
15666            HealthProbeError::bad_answer("").label(),
15667            HealthProbeError::misconfigured("").label(),
15668        ];
15669        let unique: HashSet<_> = labels.iter().collect();
15670        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15671    }
15672
15673    /// The class is additional information, not a replacement.
15674    ///
15675    /// An operator needs both "this was silence" and the specific text saying how
15676    /// long we waited; a classification that swallowed the message would trade one
15677    /// missing distinction for another.
15678    #[test]
15679    fn classification_preserves_the_original_message() {
15680        let err = HealthProbeError::no_answer("module did not answer within 5s");
15681        assert_eq!(err.to_string(), "module did not answer within 5s");
15682        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15683    }
15684}
15685
15686#[cfg(test)]
15687mod health_tombstone_tests {
15688    use std::{path::PathBuf, sync::Arc, time::Duration};
15689
15690    use subc_protocol::{
15691        manifest::Concurrency,
15692        session::{HealthStatus, ModuleControlResponse},
15693    };
15694    use tokio::sync::mpsc;
15695
15696    use super::{
15697        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15698        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15699    };
15700    use crate::{
15701        control::ControlHandler,
15702        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15703        registry::{ConnectionId, Registry},
15704        router::FrameSink,
15705    };
15706
15707    struct ProbeHarness {
15708        spec: ModuleSpec,
15709        runtime: SupervisorRuntimeConfig,
15710        forwarding: Arc<ForwardingTable>,
15711        module_connection: ConnectionId,
15712        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15713        handler: ControlHandler,
15714        module: super::SupervisedModule,
15715    }
15716
15717    fn probe_harness() -> ProbeHarness {
15718        let registry = Arc::new(Registry::default());
15719        let forwarding = Arc::new(ForwardingTable::default());
15720        let supervisor_handle = super::SupervisorHandle::new();
15721        let health = HealthConfig {
15722            http: None,
15723            cadence: Duration::from_secs(30),
15724            deadline: Duration::from_secs(5),
15725            failure_threshold: 3,
15726            on_degraded: HealthAction::Report,
15727            on_failing: HealthAction::Report,
15728            critical: false,
15729        };
15730        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
15731            .with_forwarding(Arc::clone(&forwarding))
15732            .with_handle(supervisor_handle.clone())
15733            .with_health_config(health);
15734        let spec = ModuleSpec {
15735            module_id: "late-health-module".to_string(),
15736            program: PathBuf::from("disabled-module"),
15737            args: Vec::new(),
15738            env: Vec::new(),
15739            reserved: false,
15740            reserved_prefixes: Vec::new(),
15741            protocol: ModuleProtocol::Subc,
15742            overlap: Default::default(),
15743        };
15744        let module = supervisor
15745            .supervise_configured(spec.clone(), false)
15746            .unwrap();
15747        let runtime = supervisor.runtime_config();
15748        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15749            .with_supervisor(supervisor_handle);
15750        let module_connection = ConnectionId::new(700);
15751        let (module_tx, module_rx) = mpsc::channel(8);
15752        forwarding
15753            .register_module_connection(
15754                module_connection,
15755                spec.module_id.clone(),
15756                subc_protocol::PROTOCOL_VERSION,
15757                Concurrency::ModuleManaged,
15758                FrameSink::new(module_tx),
15759            )
15760            .unwrap();
15761
15762        ProbeHarness {
15763            spec,
15764            runtime,
15765            forwarding,
15766            module_connection,
15767            module_rx,
15768            handler,
15769            module,
15770        }
15771    }
15772
15773    async fn finish_after(
15774        harness: &mut ProbeHarness,
15775        stall: Duration,
15776    ) -> ModuleControlRpcCompletion {
15777        assert!(stall > harness.runtime.health.deadline);
15778        let deadline = harness.runtime.health.deadline;
15779        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15780        let answer = async {
15781            let frame = harness.module_rx.recv().await.expect("health.check frame");
15782            tokio::time::advance(deadline).await;
15783            tokio::task::yield_now().await;
15784            tokio::time::advance(stall - deadline).await;
15785            harness
15786                .forwarding
15787                .complete_module_control_rpc(
15788                    harness.module_connection,
15789                    frame.header.corr,
15790                    Some("health.check"),
15791                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15792                        status: HealthStatus::Ok,
15793                        detail: None,
15794                        metrics: None,
15795                    }),
15796                )
15797                .unwrap()
15798        };
15799        let (probe_result, completion) = tokio::join!(probe, answer);
15800        let err = probe_result.expect_err("probe must miss its deadline");
15801        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15802        completion
15803    }
15804
15805    async fn time_out_without_answer(harness: &mut ProbeHarness) {
15806        let deadline = harness.runtime.health.deadline;
15807        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15808        let exhaust_deadline = async {
15809            let _frame = harness.module_rx.recv().await.expect("health.check frame");
15810            tokio::time::advance(deadline).await;
15811            tokio::task::yield_now().await;
15812        };
15813        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15814        let err = probe_result.expect_err("probe must miss its deadline");
15815        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15816    }
15817
15818    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15819        let registry = Arc::clone(&harness.module.inner.registry);
15820        let snapshot = Arc::clone(&harness.module.inner.snapshot);
15821        let process_liveness = super::SupervisorProcessLiveness::default();
15822        let mut child = None;
15823        let cycle = super::run_health_probe_cycle(
15824            &harness.spec,
15825            &harness.runtime,
15826            &registry,
15827            &process_liveness,
15828            &snapshot,
15829            &mut child,
15830        );
15831        let peer = async {
15832            let frame = harness.module_rx.recv().await.expect("health.check frame");
15833            if answer {
15834                harness
15835                    .forwarding
15836                    .complete_module_control_rpc(
15837                        harness.module_connection,
15838                        frame.header.corr,
15839                        Some("health.check"),
15840                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15841                            status: HealthStatus::Ok,
15842                            detail: None,
15843                            metrics: Some(serde_json::json!({"ready": true})),
15844                        }),
15845                    )
15846                    .unwrap();
15847            } else {
15848                tokio::time::advance(harness.runtime.health.deadline).await;
15849                tokio::task::yield_now().await;
15850            }
15851        };
15852        tokio::join!(cycle, peer);
15853    }
15854
15855    #[tokio::test(start_paused = true)]
15856    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15857        let mut harness = probe_harness();
15858        // Drive the probe cycle directly with an in-memory wire peer. Stop the
15859        // disabled module's monitor so only this test owns lifecycle transitions;
15860        // no OS process is launched, and a restart is observed at scheduling.
15861        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15862        monitor.abort();
15863        let _ = monitor.await;
15864        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15865            state.enabled = true;
15866            state.state = super::ModuleState::Running;
15867            state.process_alive = true;
15868        })
15869        .unwrap();
15870
15871        run_probe_cycle(&mut harness, true).await;
15872        assert_eq!(
15873            harness.module.status().unwrap().health.status,
15874            super::SupervisorHealthStatus::Ok
15875        );
15876
15877        for failures in 1..harness.runtime.health.failure_threshold {
15878            run_probe_cycle(&mut harness, false).await;
15879            let status = harness.module.status().unwrap();
15880            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15881            assert_eq!(status.health.consecutive_failures, failures);
15882            assert!(status.health.last_probe_ms.is_some());
15883            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15884            assert!(status.health.metrics.is_none());
15885            assert_eq!(status.state, super::ModuleState::Running);
15886            assert!(status.process_alive);
15887            assert_eq!(status.restart_count, 0);
15888            assert_eq!(status.lifetime_restarts, 0);
15889            assert!(status.health.last_action.is_none());
15890            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15891        }
15892
15893        run_probe_cycle(&mut harness, true).await;
15894        let recovered = harness.module.status().unwrap();
15895        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15896        assert_eq!(recovered.health.consecutive_failures, 0);
15897        assert!(recovered.health.detail.is_none());
15898        assert_eq!(
15899            recovered.health.metrics,
15900            Some(serde_json::json!({"ready": true}))
15901        );
15902        assert_eq!(recovered.lifetime_restarts, 0);
15903
15904        for failures in 1..=harness.runtime.health.failure_threshold {
15905            run_probe_cycle(&mut harness, false).await;
15906            let status = harness.module.status().unwrap();
15907            assert_eq!(status.health.consecutive_failures, failures);
15908            if failures < harness.runtime.health.failure_threshold {
15909                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15910                assert_eq!(status.state, super::ModuleState::Running);
15911                assert_eq!(status.lifetime_restarts, 0);
15912                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15913            } else {
15914                assert_eq!(
15915                    status.health.status,
15916                    super::SupervisorHealthStatus::Unresponsive
15917                );
15918                assert_eq!(status.state, super::ModuleState::Restarting);
15919                assert_eq!(status.restart_count, 1);
15920                assert_eq!(status.lifetime_restarts, 1);
15921                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15922                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15923            }
15924        }
15925    }
15926
15927    #[tokio::test(start_paused = true)]
15928    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15929        let mut harness = probe_harness();
15930
15931        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15932        let first_latency = match &first {
15933            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15934            other => panic!("late answer was not retained: {other:?}"),
15935        };
15936        assert!(harness.handler.observe_module_control_completion(first));
15937
15938        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15939        let second_latency = match &second {
15940            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15941            other => panic!("late answer was not retained: {other:?}"),
15942        };
15943        assert!(harness.handler.observe_module_control_completion(second));
15944
15945        assert_eq!(first_latency, Duration::from_secs(8));
15946        assert_eq!(
15947            second_latency - first_latency,
15948            Duration::from_secs(3),
15949            "latency must grow linearly with the additional stall"
15950        );
15951        let health = harness.module.status().unwrap().health;
15952        assert_eq!(health.late_answer_count, 2);
15953        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15954    }
15955
15956    /// A module that answers every probe late must never march to the kill
15957    /// threshold: the late answer proves it is alive, so it must clear the miss
15958    /// streak the timeout recorded. Without the reset, a CPU-starved module
15959    /// that serves every probe seconds past the deadline accumulates
15960    /// `consecutive_failures` to the threshold and is killed — the exact
15961    /// sequence from the 2026-08-14 aft disable, where the daemon logged
15962    /// "proves the module is alive" five times while counting five misses.
15963    #[tokio::test(start_paused = true)]
15964    async fn late_answer_clears_the_consecutive_failure_streak() {
15965        let mut harness = probe_harness();
15966
15967        // Timeout recorded first: the probe path saw no answer in time.
15968        time_out_without_answer(&mut harness).await;
15969        harness
15970            .module
15971            .record_health_probe_failure_for_test("[no-answer] test miss")
15972            .unwrap();
15973        assert_eq!(
15974            harness.module.status().unwrap().health.consecutive_failures,
15975            1,
15976            "precondition: the miss must be on the streak before the late answer"
15977        );
15978
15979        // The stalled reply then lands: proof of life.
15980        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15981        assert!(matches!(
15982            late,
15983            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15984        ));
15985        assert!(harness.handler.observe_module_control_completion(late));
15986
15987        let health = harness.module.status().unwrap().health;
15988        assert_eq!(
15989            health.consecutive_failures, 0,
15990            "a late answer is an answer: the streak must reset"
15991        );
15992        assert_eq!(health.late_answer_count, 1);
15993    }
15994
15995    #[tokio::test(start_paused = true)]
15996    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15997        let mut harness = probe_harness();
15998
15999        for _ in 0..20 {
16000            time_out_without_answer(&mut harness).await;
16001            assert_eq!(
16002                harness.forwarding.health_probe_tombstone_count().unwrap(),
16003                1
16004            );
16005        }
16006    }
16007}
16008
16009#[cfg(test)]
16010mod child_env_tests {
16011    #[cfg(windows)]
16012    use super::{
16013        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16014    };
16015    #[cfg(windows)]
16016    use std::sync::{Arc, OnceLock};
16017    #[cfg(windows)]
16018    #[test]
16019    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16020        let spec = spec(Vec::new());
16021        let mut command = Command::new("unused");
16022        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16023            .unwrap()
16024            .unwrap();
16025        let name = command
16026            .as_std()
16027            .get_envs()
16028            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16029            .and_then(|(_, value)| value)
16030            .unwrap();
16031        assert_eq!(name, OsStr::new(handoff.name()));
16032        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16033        assert!(command.as_std().get_envs().any(|(key, value)| key
16034            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16035            && value == Some(OsStr::new("env"))));
16036        assert!(command
16037            .as_std()
16038            .get_envs()
16039            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16040    }
16041
16042    #[cfg(windows)]
16043    #[test]
16044    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16045        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16046        snapshot.process_alive = true;
16047        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16048        assert_eq!(
16049            observed_launch_nonce_source(&snapshot).as_deref(),
16050            Some("env")
16051        );
16052        let consumed = Arc::new(OnceLock::new());
16053        snapshot.pipe_consumed = Some(consumed.clone());
16054        assert_eq!(
16055            observed_launch_nonce_source(&snapshot).as_deref(),
16056            Some("env")
16057        );
16058        consumed.set(()).unwrap();
16059        assert_eq!(
16060            observed_launch_nonce_source(&snapshot).as_deref(),
16061            Some("pipe")
16062        );
16063        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16064        assert_eq!(
16065            observed_launch_nonce_source(&snapshot).as_deref(),
16066            Some("env")
16067        );
16068        clear_current_process_facts(&mut snapshot);
16069        assert_eq!(observed_launch_nonce_source(&snapshot), None);
16070    }
16071    use super::{
16072        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16073        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16074        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16075    };
16076    use std::{ffi::OsStr, path::PathBuf};
16077    use tokio::process::Command;
16078
16079    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16080        ModuleSpec {
16081            module_id: "env-plan".to_string(),
16082            program: PathBuf::from("/nonexistent"),
16083            args: Vec::new(),
16084            env,
16085            reserved: false,
16086            reserved_prefixes: Vec::new(),
16087            protocol: ModuleProtocol::Subc,
16088            overlap: Default::default(),
16089        }
16090    }
16091
16092    #[test]
16093    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16094        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16095        let digest = LaunchSpecDigest::from_spec(&base);
16096        let mut reordered = base.clone();
16097        reordered.env.reverse();
16098        reordered.module_id = "different-id".into();
16099        reordered.reserved = true;
16100        reordered.overlap = Default::default();
16101        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16102        let mut delimiter_collision = base.clone();
16103        delimiter_collision.env[0] = ("A=B".into(), "C".into());
16104        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16105        let mut changed = base.clone();
16106        changed.protocol = ModuleProtocol::None;
16107        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16108        changed = base.clone();
16109        changed.args.push("--flag".into());
16110        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16111        changed = base.clone();
16112        changed.program = PathBuf::from("/other");
16113        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16114        changed = base.clone();
16115        changed.env[1].1 = "other".into();
16116        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16117        assert!(!format!("{digest:?}").contains("B=C"));
16118        assert!(!format!("{digest:?}").contains("[u8"));
16119    }
16120
16121    #[cfg(unix)]
16122    #[test]
16123    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16124        use std::os::unix::ffi::OsStringExt;
16125        let mut left = spec(Vec::new());
16126        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16127        let mut right = left.clone();
16128        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16129        assert_eq!(
16130            left.program.to_string_lossy(),
16131            right.program.to_string_lossy()
16132        );
16133        assert_ne!(
16134            LaunchSpecDigest::from_spec(&left),
16135            LaunchSpecDigest::from_spec(&right)
16136        );
16137    }
16138
16139    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
16140    /// one still gets its own.
16141    ///
16142    /// This is the narrow goal `env_clear()` was reached for, and the reason the
16143    /// fix is `env_remove` rather than deleting the line: an operator's ambient
16144    /// filter silently becoming an unconfigured module's log level is a real
16145    /// defect, just a much smaller one than clearing the environment.
16146    ///
16147    /// Asserted on the command plan rather than a spawned child because proving
16148    /// the ABSENCE of an inherited variable needs the parent's environment
16149    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
16150    /// removal as `(key, None)`, which is exactly the distinction wanted: not
16151    /// "absent because nobody set it" but "explicitly unset for the child".
16152    #[test]
16153    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16154        let mut command = Command::new("/nonexistent");
16155        apply_child_env(&mut command, &spec(Vec::new()));
16156        let removed = command
16157            .as_std()
16158            .get_envs()
16159            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16160        assert!(
16161            removed,
16162            "ambient CK_LOG must be explicitly removed for an unconfigured module"
16163        );
16164
16165        let mut configured = Command::new("/nonexistent");
16166        apply_child_env(
16167            &mut configured,
16168            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16169        );
16170        let effective = configured
16171            .as_std()
16172            .get_envs()
16173            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16174            .last()
16175            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16176        assert_eq!(
16177            effective,
16178            Some(Some("debug".to_string())),
16179            "a module's configured CK_LOG must survive the ambient removal"
16180        );
16181    }
16182
16183    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
16184    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
16185    /// the same reason as the CK_LOG test above.
16186    ///
16187    /// The argument is the load-bearing half: a stock binary exits on an
16188    /// unknown flag before it listens, so with `--subc` appended the mode
16189    /// could not supervise the one process it exists for. Found by the first
16190    /// conformance run (nats-server: `flag provided but not defined: -subc`).
16191    #[test]
16192    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16193        let connection_file = std::path::Path::new("/run/subc-connection.json");
16194        let handle = SupervisorHandle::new();
16195
16196        let mut none_spec = spec(Vec::new());
16197        none_spec.protocol = ModuleProtocol::None;
16198        let mut none = Command::new("/nonexistent");
16199        let none_handoff =
16200            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16201                .expect("protocol-none spawn args apply");
16202        assert!(
16203            none_handoff.is_none(),
16204            "protocol:none spawn must not receive a nonce descriptor"
16205        );
16206        #[cfg(windows)]
16207        assert!(!none.as_std().get_envs().any(|(key, value)| key
16208            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16209            && value.is_some()));
16210        assert!(
16211            !none.as_std().get_envs().any(|(key, value)| key
16212                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16213                && value.is_some()),
16214            "protocol:none spawn must not name a nonce descriptor"
16215        );
16216        let none_args: Vec<String> = none
16217            .as_std()
16218            .get_args()
16219            .map(|a| a.to_string_lossy().into_owned())
16220            .collect();
16221        assert!(
16222            !none_args.iter().any(|a| a == SUBC_ARG),
16223            "protocol:none argv must not carry --subc; got {none_args:?}"
16224        );
16225        let none_has_nonce = none
16226            .as_std()
16227            .get_envs()
16228            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16229        assert!(
16230            !none_has_nonce,
16231            "protocol:none spawn must not receive a launch nonce"
16232        );
16233        let none_has_module_id = none
16234            .as_std()
16235            .get_envs()
16236            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16237        assert!(
16238            none_has_module_id,
16239            "SUBC_MODULE_ID is inert and stays on every path"
16240        );
16241        assert!(
16242            handle.spawn_nonce(&none_spec.module_id).is_none(),
16243            "no nonce record for a process that will never present one"
16244        );
16245
16246        // Control: the subc-wire path is unchanged by the branch above.
16247        let wire_spec = spec(Vec::new());
16248        let mut wire = Command::new("/nonexistent");
16249        let wire_handoff =
16250            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16251                .expect("subc-wire spawn args apply");
16252        let wire_fd_env = wire
16253            .as_std()
16254            .get_envs()
16255            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16256            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16257        #[cfg(unix)]
16258        assert_eq!(
16259            wire_fd_env,
16260            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16261            "a subc-wire spawn names the pipe it will receive at descriptor 3"
16262        );
16263        #[cfg(windows)]
16264        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16265        #[cfg(not(any(unix, windows)))]
16266        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16267        let wire_args: Vec<String> = wire
16268            .as_std()
16269            .get_args()
16270            .map(|a| a.to_string_lossy().into_owned())
16271            .collect();
16272        assert_eq!(
16273            wire_args,
16274            vec![
16275                SUBC_ARG.to_string(),
16276                connection_file.to_string_lossy().into_owned()
16277            ],
16278            "a subc-wire spawn still carries --subc <path>"
16279        );
16280        assert_eq!(
16281            wire.as_std()
16282                .get_envs()
16283                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16284            !cfg!(unix),
16285            "only Windows supplies the environment nonce"
16286        );
16287        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16288    }
16289
16290    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
16291    /// spec tries to set it; only a swap candidate carries it.
16292    ///
16293    /// "Set it only on candidates" is not enough, because spawn applies the
16294    /// spec's env verbatim and the daemon's own environment is inherited: either
16295    /// could hand a plain restart the swap role, and a module reading it would
16296    /// warm on its long swap budget while callers wait. Asserted as an explicit
16297    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
16298    /// test above gives.
16299    #[test]
16300    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16301        let role = |command: &Command| {
16302            command
16303                .as_std()
16304                .get_envs()
16305                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16306                .last()
16307                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16308        };
16309        let forged = spec(vec![(
16310            SUBC_SPAWN_ROLE_ENV.to_string(),
16311            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16312        )]);
16313
16314        let mut plain = Command::new("/nonexistent");
16315        apply_child_env(&mut plain, &forged);
16316        apply_spawn_role(&mut plain, SpawnRole::Plain);
16317        assert_eq!(
16318            role(&plain),
16319            Some(None),
16320            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16321        );
16322
16323        let mut candidate = Command::new("/nonexistent");
16324        apply_child_env(&mut candidate, &spec(Vec::new()));
16325        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16326        assert_eq!(
16327            role(&candidate),
16328            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16329        );
16330    }
16331
16332    /// Daemon-private capture retention keys never reach the child.
16333    ///
16334    /// cortexkit-log exposes retention as a Rust struct with no environment
16335    /// names, so these entries are supervisor metadata. Passing them through
16336    /// would invent a public child-process contract by accident.
16337    #[test]
16338    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16339        let mut command = Command::new("/nonexistent");
16340        apply_child_env(
16341            &mut command,
16342            &spec(vec![
16343                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16344                ("KEPT".to_string(), "yes".to_string()),
16345            ]),
16346        );
16347        let keys: Vec<String> = command
16348            .as_std()
16349            .get_envs()
16350            .filter(|(_, value)| value.is_some())
16351            .map(|(key, _)| key.to_string_lossy().into_owned())
16352            .collect();
16353        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16354        assert!(
16355            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16356            "daemon-private capture key leaked to the child: {keys:?}"
16357        );
16358    }
16359}
16360
16361#[cfg(test)]
16362mod jitter_tests {
16363    use super::jittered_health_delay;
16364    use std::{collections::HashSet, time::Duration};
16365
16366    /// Module ids drawn from a real fleet, so the dispersal claim is about names
16367    /// that actually occur rather than invented ones.
16368    ///
16369    /// This is a SAMPLE, not a registry: the property under test is that distinct
16370    /// ids disperse, which holds for any set of distinct strings. Several entries
16371    /// are already historical (modules get renamed), and that costs nothing here --
16372    /// but it means a reader must not mistake this for the live module set, and a
16373    /// rename sweep will match it without there being anything to change.
16374    const FLEET: [&str; 14] = [
16375        "aft",
16376        "alfonso-core",
16377        "magic-context",
16378        "broca",
16379        "thalamus",
16380        "quota",
16381        "engram",
16382        "plexus",
16383        "cerebellum",
16384        "astrocyte",
16385        "synapse",
16386        "subc-mcp",
16387        "cortexkit-credentials",
16388        "subc-federation",
16389    ];
16390
16391    /// Probes must not converge after a fleet-wide restart.
16392    ///
16393    /// This is the property the jitter exists for: every module reconnects at
16394    /// once, and without dispersal all fourteen would then probe on the same
16395    /// tick forever. Nothing failed visibly when this went untested -- a
16396    /// convergent fleet still probes correctly, just in a burst, so the symptom
16397    /// is a periodic load spike that looks like whatever else is running.
16398    #[test]
16399    fn probe_delays_disperse_across_the_fleet() {
16400        let cadence = Duration::from_secs(30);
16401        let delays: HashSet<Duration> = FLEET
16402            .iter()
16403            .map(|id| jittered_health_delay(id, 0, cadence))
16404            .collect();
16405        assert_eq!(
16406            delays.len(),
16407            FLEET.len(),
16408            "every supervised module must land on its own probe offset"
16409        );
16410    }
16411
16412    /// The offset may only ever DELAY a probe, never bring it forward.
16413    ///
16414    /// A delay below the cadence would probe a module more often than
16415    /// configured, which is the opposite of what an operator asked for and
16416    /// would tighten the failure budget without anyone changing it.
16417    #[test]
16418    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16419        let cadence = Duration::from_secs(30);
16420        let span = cadence / 10;
16421        for id in FLEET {
16422            for probe_index in 0..8 {
16423                let delay = jittered_health_delay(id, probe_index, cadence);
16424                assert!(
16425                    delay >= cadence,
16426                    "{id}#{probe_index}: jitter must not shorten the cadence"
16427                );
16428                assert!(
16429                    delay < cadence + span,
16430                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16431                );
16432            }
16433        }
16434    }
16435
16436    /// A module keeps its offset across daemon restarts.
16437    ///
16438    /// The delay is derived rather than randomised precisely so a restart does
16439    /// not re-roll every module into a fresh chance of collision. A random
16440    /// source would satisfy the dispersal test above and quietly lose this.
16441    #[test]
16442    fn a_module_offset_is_stable_across_restarts() {
16443        let cadence = Duration::from_secs(30);
16444        for id in FLEET {
16445            assert_eq!(
16446                jittered_health_delay(id, 0, cadence),
16447                jittered_health_delay(id, 0, cadence),
16448                "{id}: the same module and probe index must produce the same offset"
16449            );
16450        }
16451    }
16452
16453    /// A zero cadence disables probing rather than producing a busy loop.
16454    #[test]
16455    fn zero_cadence_yields_zero_delay() {
16456        assert_eq!(
16457            jittered_health_delay("aft", 0, Duration::ZERO),
16458            Duration::ZERO
16459        );
16460    }
16461}
16462
16463#[cfg(all(test, target_os = "linux"))]
16464mod cgroup_placement_tests {
16465    use super::{
16466        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16467        SuperviseError, SupervisedChild,
16468    };
16469    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16470    use cortexkit_test_support::ScratchDir;
16471    use std::{
16472        fs, io,
16473        path::{Path, PathBuf},
16474        sync::{Arc, Mutex},
16475    };
16476    use tokio::process::Command;
16477
16478    #[tokio::test]
16479    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16480        use super::*;
16481        let dir = ScratchDir::new("unique-spawn-cgroups");
16482        let root = PathBuf::from(format!(
16483            "/sys/fs/cgroup/subc-unique-test-{}-{}",
16484            std::process::id(),
16485            unix_ms_now()
16486        ));
16487        if let Err(error) = fs::create_dir(&root) {
16488            assert!(
16489                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16490                "required cgroup test cannot execute: {error}"
16491            );
16492            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16493            return;
16494        }
16495        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16496        let group_count = || {
16497            fs::read_dir(root.join("subc-modules"))
16498                .unwrap()
16499                .map(|entry| entry.unwrap().file_type().unwrap())
16500                .filter(|kind| kind.is_dir())
16501                .count()
16502        };
16503        let supervisor =
16504            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16505                .with_cgroup_placement(Some(placement.clone()));
16506        let runtime = supervisor.runtime_config();
16507        let mut spec = ModuleSpec {
16508            module_id: "unique-spawn".into(),
16509            program: PathBuf::from("/bin/sleep"),
16510            args: vec!["60".into()],
16511            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16512                .into_iter()
16513                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16514                .collect(),
16515            reserved: false,
16516            reserved_prefixes: vec![],
16517            protocol: ModuleProtocol::None,
16518            overlap: Default::default(),
16519        };
16520        let spawn = |spec: &ModuleSpec| {
16521            spawn_child(
16522                spec,
16523                None,
16524                None,
16525                &runtime.stderr_ring,
16526                None,
16527                &runtime.child_roster,
16528                Some(&placement),
16529            )
16530            .unwrap()
16531        };
16532        let mut live = spawn(&spec);
16533        for _ in 0..3 {
16534            // A new process can enter the old slot while retirement is pending.
16535            let next = spawn(&spec);
16536            assert_ne!(live.module_id, next.module_id);
16537            live.start_kill().unwrap();
16538            live.wait().await.unwrap();
16539            live = next;
16540            assert!(
16541                live.child.try_wait().unwrap().is_none(),
16542                "retiring the old slot must not kill the replacement"
16543            );
16544            assert_eq!(
16545                group_count(),
16546                1,
16547                "only the live spawn's cgroup should remain"
16548            );
16549        }
16550        supervisor.begin_daemon_shutdown();
16551        let reap = tokio::spawn(async move {
16552            live.wait().await.unwrap();
16553        });
16554        supervisor
16555            .end_children_for_daemon_shutdown(false, std::future::pending())
16556            .await;
16557        reap.await.unwrap();
16558        assert_eq!(group_count(), 0);
16559        // A normal exit uses the same tree-cleanup path as a killed spawn.
16560        spec.program = PathBuf::from("/bin/true");
16561        spec.args.clear();
16562        let fresh_roster = ChildRoster::default();
16563        let mut short = spawn_child(
16564            &spec,
16565            None,
16566            None,
16567            &runtime.stderr_ring,
16568            None,
16569            &fresh_roster,
16570            Some(&placement),
16571        )
16572        .unwrap();
16573        short.wait().await.unwrap();
16574        assert_eq!(group_count(), 0);
16575        spec.module_id = "_".repeat(255);
16576        let mut long_id = spawn_child(
16577            &spec,
16578            None,
16579            None,
16580            &runtime.stderr_ring,
16581            None,
16582            &fresh_roster,
16583            Some(&placement),
16584        )
16585        .unwrap();
16586        long_id.wait().await.unwrap();
16587        assert_eq!(
16588            group_count(),
16589            0,
16590            "valid long module IDs must not exceed cgroup NAME_MAX"
16591        );
16592        fs::remove_dir(root.join("subc-modules")).unwrap();
16593        fs::remove_dir(root).unwrap();
16594        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16595    }
16596
16597    #[test]
16598    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16599        let path = Path::new("/definitely-missing-subc-cgroup");
16600        let mut command = Command::new("true");
16601        let error = apply_cgroup_placement(
16602            &mut command,
16603            &ModuleSpec {
16604                module_id: "broken-cgroup".to_string(),
16605                program: PathBuf::from("true"),
16606                args: Vec::new(),
16607                env: Vec::new(),
16608                reserved: false,
16609                reserved_prefixes: Vec::new(),
16610                protocol: ModuleProtocol::Subc,
16611                overlap: Default::default(),
16612            },
16613            path,
16614        )
16615        .expect_err("a parent cgroup open failure must reject the supervised spawn");
16616        let reason = error.to_string();
16617
16618        assert!(
16619            matches!(error, SuperviseError::Cgroup { .. }),
16620            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16621        );
16622        assert!(
16623            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16624            "parent cgroup open failure must name cgroup.procs: {reason}"
16625        );
16626    }
16627
16628    #[tokio::test]
16629    async fn reaping_a_child_removes_its_empty_module_cgroup() {
16630        let root = ScratchDir::new("supervisor-reap-cgroup");
16631        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16632        let placement = subc_cgroup::prepare_at(&root)
16633            .expect("prepare scratch cgroup root")
16634            .expect("scratch root has a cgroup.procs marker");
16635        let module_id = "reaped-module";
16636        let module = placement
16637            .module_path(module_id)
16638            .expect("create scratch module cgroup");
16639        let child = Command::new("true")
16640            .env("XDG_DATA_HOME", root.path())
16641            .env("XDG_RUNTIME_DIR", root.path())
16642            .env("XDG_CONFIG_HOME", root.path())
16643            .spawn()
16644            .expect("spawn short-lived child");
16645        let pid = child.id().expect("spawned child has pid");
16646        let mut child = SupervisedChild {
16647            subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
16648            child,
16649            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16650            protocol: ModuleProtocol::Subc,
16651            module_id: module_id.to_string(),
16652            cgroup_placement: Some(placement),
16653            stdout_pump: None,
16654            stderr_pump: None,
16655            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16656            spawned_at: tokio::time::Instant::now(),
16657            registration_deadline: None,
16658            registration_watch: None,
16659            spawned_at_ms: 0,
16660            spawned_from: PathBuf::from("true"),
16661            launch_spec_digest: LaunchSpecDigest([0; 32]),
16662            spawned_file_identity: None,
16663            process_start_time: None,
16664            process_identity: None,
16665            pid,
16666            roster_guard: None,
16667            #[cfg(target_os = "macos")]
16668            privacy_exec: None,
16669            spawn_failure: None,
16670        };
16671
16672        child.wait().await.expect("reap short-lived child");
16673
16674        assert!(
16675            !module.exists(),
16676            "reaping the supervised child must remove its empty cgroup"
16677        );
16678    }
16679
16680    #[test]
16681    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16682        let root = ScratchDir::new("supervisor-non-empty-cgroup");
16683        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16684        let placement = subc_cgroup::prepare_at(&root)
16685            .expect("prepare scratch cgroup root")
16686            .expect("scratch root has a cgroup.procs marker");
16687        let module = placement
16688            .module_path("surviving-module")
16689            .expect("create scratch module cgroup");
16690        fs::write(module.join("surviving-process"), b"still present")
16691            .expect("make scratch cgroup non-empty");
16692        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16693
16694        remove_module_cgroup(&placement, "surviving-module");
16695
16696        let logs = crate::router::test_log::captured_logs(&logs);
16697        assert!(
16698            module.exists(),
16699            "failed removal must leave the cgroup intact"
16700        );
16701        assert!(
16702            logs.contains("could not remove module cgroup after process exit; continuing teardown")
16703                && logs.contains("surviving-module"),
16704            "best-effort removal must report the failure without returning it: {logs}"
16705        );
16706    }
16707
16708    #[test]
16709    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16710        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16711        let reason = SuperviseError::Spawn {
16712            program: PathBuf::from("/bin/true"),
16713            source: io::Error::from_raw_os_error(13),
16714            cgroup_path: Some(cgroup_path.clone()),
16715        }
16716        .to_string();
16717
16718        assert!(
16719            reason.contains(&cgroup_path.display().to_string()),
16720            "a pre_exec spawn failure must name the cgroup path: {reason}"
16721        );
16722    }
16723}
16724
16725#[cfg(test)]
16726mod spawn_subscriber_lag_tests {
16727    use super::*;
16728
16729    /// A subscriber whose connection stops draining is dropped once its frame
16730    /// channel fills. The client must learn that from a terminal Error frame
16731    /// after the frames already queued for it, not from a stream that simply
16732    /// goes quiet.
16733    #[tokio::test]
16734    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16735        let feed = SpawnEventFeed::default();
16736        feed.configure_incarnation("lag-incarnation".to_string());
16737        // A one-slot connection queue that nobody reads until the emits are
16738        // done: the forwarder parks on it and the subscriber channel fills.
16739        let (tx, mut rx) = mpsc::channel(1);
16740        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16741            .expect("subscribe");
16742        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16743        for index in 0..emitted {
16744            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16745            // Let the forwarder take what it can so the fill point is the
16746            // subscriber channel, not a scheduling accident.
16747            tokio::task::yield_now().await;
16748        }
16749        assert_eq!(
16750            feed.subscriber_count(),
16751            0,
16752            "the lagged subscriber must be removed"
16753        );
16754
16755        let mut data = Vec::new();
16756        let mut last = None;
16757        loop {
16758            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16759                .await
16760                .expect("the forwarder must finish once the subscriber is dropped");
16761            let Some(outbound) = next else { break };
16762            let frame = outbound.frame;
16763            if frame.header.ty == FrameType::StreamData {
16764                assert!(last.is_none(), "no data may follow the terminal frame");
16765                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16766                data.push(event.cursor.seq);
16767            } else {
16768                assert!(last.is_none(), "exactly one terminal frame");
16769                last = Some(frame);
16770            }
16771        }
16772        assert!(!data.is_empty(), "queued frames drain before the terminal");
16773        for pair in data.windows(2) {
16774            assert_eq!(
16775                pair[1],
16776                pair[0] + 1,
16777                "queued frames arrive dense and in order"
16778            );
16779        }
16780        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16781        assert_eq!(terminal.header.ty, FrameType::Error);
16782        assert_eq!(terminal.header.corr, 7);
16783        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16784        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16785        let detail = body.detail.expect("lagged error carries detail");
16786        assert_eq!(
16787            detail["first_undelivered_cursor"]["seq"],
16788            data.last().unwrap() + 1,
16789            "the named cursor is the first event the subscriber did not receive"
16790        );
16791        assert_eq!(
16792            detail["first_undelivered_cursor"]["daemon_incarnation"],
16793            "lag-incarnation"
16794        );
16795    }
16796}
16797
16798#[cfg(test)]
16799mod terminal_history_read_concurrency_tests {
16800    use super::*;
16801    use crate::terminal_journal::read_pause;
16802    use cortexkit_test_support::ScratchDir;
16803    use std::sync::mpsc as std_mpsc;
16804
16805    fn journaled_ring(
16806        journal: &Arc<crate::terminal_journal::TerminalJournal>,
16807    ) -> Arc<Mutex<TerminalRing>> {
16808        Arc::new(Mutex::new(
16809            TerminalRing::new(TerminalRingConfig::default(), 1)
16810                .with_journal(Some(Arc::clone(journal))),
16811        ))
16812    }
16813
16814    fn crash(at_ms: u64) -> ExitReport {
16815        ExitReport {
16816            kind: ExitKind::Crash,
16817            code: Some(1),
16818            signal: None,
16819            at_ms,
16820        }
16821    }
16822
16823    /// Record an exit on another thread and report whether it finished within
16824    /// `bound`. The recorder thread is left running if it did not.
16825    fn record_within(
16826        module_id: &'static str,
16827        ring: &Arc<Mutex<TerminalRing>>,
16828        at_ms: u64,
16829        bound: Duration,
16830    ) -> bool {
16831        let ring = Arc::clone(ring);
16832        let (done, done_rx) = std_mpsc::channel();
16833        std::thread::spawn(move || {
16834            record_terminal(
16835                module_id,
16836                &ring,
16837                &SpawnEventFeed::default(),
16838                &crash(at_ms),
16839                TerminalDisposition::Restarting,
16840            );
16841            let _ = done.send(());
16842        });
16843        done_rx.recv_timeout(bound).is_ok()
16844    }
16845
16846    /// A history read in progress must not hold the journal writer (which every
16847    /// module's exit recording needs) or the module's own ring. Exits recorded
16848    /// while the read is paused complete promptly; the paused read answers as of
16849    /// the moment it started, and the next read has each exit exactly once.
16850    #[test]
16851    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16852        let dir = ScratchDir::new("terminal-history-concurrent-read");
16853        let path = dir.join("terminals.jsonl");
16854        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16855            path.clone(),
16856            "daemon".into(),
16857        ));
16858        let reader_ring = journaled_ring(&journal);
16859        let other_ring = journaled_ring(&journal);
16860        assert!(record_within(
16861            "reader-module",
16862            &reader_ring,
16863            10,
16864            Duration::from_secs(5)
16865        ));
16866
16867        let (started, release) = read_pause::install(&path);
16868        let reading = {
16869            let ring = Arc::clone(&reader_ring);
16870            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16871        };
16872        started
16873            .recv_timeout(Duration::from_secs(5))
16874            .expect("the history read reached its pause");
16875
16876        let bound = Duration::from_secs(1);
16877        assert!(
16878            record_within("other-module", &other_ring, 20, bound),
16879            "another module's exit waited on a history read (journal writer held)"
16880        );
16881        assert!(
16882            record_within("reader-module", &reader_ring, 30, bound),
16883            "the read module's own exit waited on its history read (ring held)"
16884        );
16885
16886        drop(release);
16887        let paused = reading.join().unwrap();
16888        assert_eq!(
16889            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16890            vec![10],
16891            "an exit recorded after the read began lands in neither half of it"
16892        );
16893        assert_eq!(paused.journal_skipped_lines, 0);
16894        assert_eq!(paused.journal_read_errors, 0);
16895
16896        let after = durable_terminal_history_of(&reader_ring, "reader-module");
16897        assert_eq!(
16898            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16899            vec![10, 30],
16900            "the next read merges ring and journal with no duplicate"
16901        );
16902        assert_eq!(after.journal_skipped_lines, 0);
16903    }
16904}
16905
16906/// What a restart does with the exited process's output readers. These drive
16907/// the same settle functions the supervisor calls, with a reader the test
16908/// holds, so a reader that has not been scheduled by the bound is a controlled
16909/// input rather than something only a loaded machine produces.
16910#[cfg(test)]
16911mod stderr_settle_tests {
16912    use std::{
16913        future::Future,
16914        io,
16915        pin::Pin,
16916        sync::{Arc, Mutex},
16917        task::{Context, Poll},
16918        time::Duration,
16919    };
16920
16921    use tokio::{
16922        io::{AsyncRead, ReadBuf},
16923        sync::oneshot,
16924        time::Instant,
16925    };
16926
16927    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16928    use crate::stderr_tail::{
16929        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16930        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16931    };
16932
16933    const BOUND: Duration = Duration::from_millis(250);
16934
16935    /// Yields `before`, then stays pending until the gate is released, then
16936    /// yields `after` and reaches EOF. The bytes after the gate were written
16937    /// by a process that has already exited; only the reader is behind.
16938    struct HeldReader {
16939        before: Option<Vec<u8>>,
16940        gate: Option<oneshot::Receiver<()>>,
16941        after: io::Cursor<Vec<u8>>,
16942    }
16943
16944    impl AsyncRead for HeldReader {
16945        fn poll_read(
16946            mut self: Pin<&mut Self>,
16947            cx: &mut Context<'_>,
16948            buf: &mut ReadBuf<'_>,
16949        ) -> Poll<io::Result<()>> {
16950            if let Some(bytes) = self.before.take() {
16951                buf.put_slice(&bytes);
16952                return Poll::Ready(Ok(()));
16953            }
16954            if let Some(gate) = self.gate.as_mut() {
16955                match Pin::new(gate).poll(cx) {
16956                    Poll::Pending => return Poll::Pending,
16957                    Poll::Ready(_) => self.gate = None,
16958                }
16959            }
16960            Pin::new(&mut self.after).poll_read(cx, buf)
16961        }
16962    }
16963
16964    struct DiscardSink;
16965
16966    impl OutputSink for DiscardSink {
16967        fn write_line(&mut self, _line: &[u8]) {}
16968    }
16969
16970    fn line(text: &str) -> TailEntry {
16971        TailEntry::Line {
16972            text: text.to_string(),
16973            truncated: false,
16974            at_ms: None,
16975        }
16976    }
16977
16978    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16979        ring.lock().unwrap()
16980    }
16981
16982    /// Start a reader for a new process generation that delivers `before`
16983    /// immediately and `after` only once the returned sender fires (or is
16984    /// dropped).
16985    fn held_pump(
16986        ring: &Arc<Mutex<StderrRing>>,
16987        before: &str,
16988        after: &str,
16989    ) -> (StderrPump, oneshot::Sender<()>) {
16990        let generation = lock(ring).begin_process();
16991        let (release, gate) = oneshot::channel();
16992        let reader = HeldReader {
16993            before: Some(before.as_bytes().to_vec()),
16994            gate: Some(gate),
16995            after: io::Cursor::new(after.as_bytes().to_vec()),
16996        };
16997        let task = tokio::spawn(pump_stderr_to(
16998            reader,
16999            Arc::clone(ring),
17000            generation,
17001            DiscardSink,
17002        ));
17003        (StderrPump { task, generation }, release)
17004    }
17005
17006    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17007        for _ in 0..1000 {
17008            if done(&lock(ring)) {
17009                return;
17010            }
17011            tokio::time::sleep(Duration::from_millis(1)).await;
17012        }
17013        panic!(
17014            "ring never reached the expected state: {:?}",
17015            lock(ring).snapshot(None, None)
17016        );
17017    }
17018
17019    #[tokio::test(start_paused = true)]
17020    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17021        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17022        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17023
17024        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17025        let before_release = lock(&ring).snapshot(None, None);
17026        assert!(
17027            matches!(before_release.capture, CaptureState::Incomplete { .. }),
17028            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17029        );
17030
17031        // The restart: the next process starts and writes before the old
17032        // reader catches up.
17033        let next = lock(&ring).begin_process();
17034        lock(&ring).push_line_from(next, "next process booting");
17035        release.send(()).unwrap();
17036        wait_until(&ring, |ring| {
17037            ring.snapshot(None, None).capture == CaptureState::Captured
17038        })
17039        .await;
17040
17041        assert_eq!(
17042            untimed(lock(&ring).snapshot(None, None).entries),
17043            vec![
17044                line("booting"),
17045                line("config error: missing storage"),
17046                TailEntry::ProcessStart,
17047                line("next process booting"),
17048            ],
17049            "the crash's last line must survive a slow reader and stay in the crashed process's section"
17050        );
17051    }
17052
17053    #[tokio::test(start_paused = true)]
17054    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17055    ) {
17056        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17057        // `_held` is never fired: a descendant keeps the pipe open for the
17058        // whole test.
17059        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17060
17061        let started = Instant::now();
17062        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17063        assert_eq!(
17064            started.elapsed(),
17065            BOUND,
17066            "the restart must wait exactly the bound for a pipe that stays open, no longer"
17067        );
17068
17069        let next = lock(&ring).begin_process();
17070        lock(&ring).push_line_from(next, "next process booting");
17071        tokio::time::sleep(Duration::from_secs(60)).await;
17072
17073        let snapshot = lock(&ring).snapshot(None, None);
17074        match &snapshot.capture {
17075            CaptureState::Incomplete { reason } => assert!(
17076                reason.contains("had not reached EOF") && reason.contains("250ms"),
17077                "the reason must say what is missing and after how long: {reason}"
17078            ),
17079            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17080        }
17081        assert_eq!(
17082            untimed(snapshot.entries),
17083            vec![
17084                line("parent exiting"),
17085                TailEntry::ProcessStart,
17086                line("next process booting"),
17087            ]
17088        );
17089    }
17090
17091    #[tokio::test(start_paused = true)]
17092    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17093        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17094        let (pump, release) = held_pump(&ring, "one\n", "two\n");
17095        release.send(()).unwrap();
17096
17097        settle_stderr_pump("clean", &ring, pump, BOUND).await;
17098
17099        let snapshot = lock(&ring).snapshot(None, None);
17100        assert_eq!(snapshot.capture, CaptureState::Captured);
17101        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17102    }
17103
17104    #[tokio::test(start_paused = true)]
17105    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17106        const BURST: usize = 150;
17107        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17108        let path = capture.join("two-pipe.stderr.log");
17109        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17110        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17111        let generation = lock(&ring).begin_process();
17112        let stdout = (0..BURST)
17113            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17114            .collect::<String>();
17115        let stderr = (0..BURST)
17116            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17117            .collect::<String>();
17118        // Model a child that has written both bursts and exited. Hold the stdout
17119        // reader after one line and a partial prefix, as if pipe readiness or
17120        // capture-file writes had delayed it. The final line also needs EOF to
17121        // flush: neither complete nor partial buffered output may be discarded.
17122        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17123        let (release, gate) = oneshot::channel();
17124        let reader = HeldReader {
17125            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17126            gate: Some(gate),
17127            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17128        };
17129        let (finished, completion) = oneshot::channel();
17130        let stdout_sink = sink.clone();
17131        let pump = tokio::spawn(async move {
17132            pump_stdout_to(reader, stdout_sink).await;
17133            finished.send(()).unwrap();
17134        });
17135        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17136
17137        let started = Instant::now();
17138        settle_stdout_pump("two-pipe", pump, BOUND).await;
17139        assert_eq!(
17140            started.elapsed(),
17141            BOUND,
17142            "a late reader must not hold up restart"
17143        );
17144        assert_eq!(
17145            std::fs::read_to_string(&path).unwrap().lines().count(),
17146            BURST + 1
17147        );
17148
17149        release
17150            .send(())
17151            .expect("stdout reader must survive the exit drain bound");
17152        tokio::time::timeout(Duration::from_secs(1), completion)
17153            .await
17154            .expect("released stdout reader must reach EOF")
17155            .expect("stdout capture must finish at EOF, not be cancelled");
17156        let contents = std::fs::read_to_string(&path).unwrap();
17157        let mut lines = contents
17158            .lines()
17159            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17160            .collect::<Vec<_>>();
17161        let mut expected = (0..BURST)
17162            .flat_map(|i| {
17163                [
17164                    format!("out-{i:04}-{}", "o".repeat(64)),
17165                    format!("err-{i:04}-{}", "e".repeat(64)),
17166                ]
17167            })
17168            .collect::<Vec<_>>();
17169        lines.sort_unstable();
17170        expected.sort_unstable();
17171        assert_eq!(
17172            lines, expected,
17173            "both pipes must retain every complete line exactly once"
17174        );
17175    }
17176}
17177
17178/// Containment of a module's process tree (issue #109).
17179///
17180/// The behaviour these defend against is a module helper surviving its module:
17181/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
17182/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
17183/// compounds it.
17184///
17185/// They run against the SUPERVISOR rather than the job-object crate because the
17186/// claim is about teardown: a crate-level test proves a job can reap a tree, not
17187/// that the daemon's drain path reaches it.
17188///
17189/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
17190/// lane there is a separate containment path with its own tests.
17191#[cfg(all(test, windows))]
17192mod job_containment_tests {
17193    use super::*;
17194    use cortexkit_test_support::ScratchDir;
17195    use std::{
17196        path::{Path, PathBuf},
17197        sync::{Arc, Mutex},
17198        time::{Duration, Instant},
17199    };
17200
17201    /// The stub, expected beside this test executable.
17202    ///
17203    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
17204    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
17205    /// failure then reads as a broken test rather than an unbuilt dependency.
17206    fn stub_path() -> PathBuf {
17207        let mut path = std::env::current_exe().expect("current_exe available in tests");
17208        path.pop();
17209        path.pop();
17210        path.push("fake-aft-stub.exe");
17211        assert!(
17212            path.exists(),
17213            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17214             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17215            path.display()
17216        );
17217        path
17218    }
17219
17220    /// Poll for the grandchild pid the stub records, and parse it.
17221    fn read_grandchild_pid(path: &Path) -> u32 {
17222        let deadline = Instant::now() + Duration::from_secs(10);
17223        loop {
17224            if let Ok(contents) = std::fs::read_to_string(path) {
17225                if let Ok(pid) = contents.trim().parse() {
17226                    return pid;
17227                }
17228            }
17229            assert!(
17230                Instant::now() < deadline,
17231                "the stub never recorded a grandchild pid at {}",
17232                path.display()
17233            );
17234            std::thread::sleep(Duration::from_millis(10));
17235        }
17236    }
17237
17238    /// Everything one fixture run needs, so the two tests below differ in exactly
17239    /// one place: whether the child is contained.
17240    struct Fixture {
17241        _dir: ScratchDir,
17242        module_id: String,
17243        grandchild: u32,
17244        child: Option<SupervisedChild>,
17245        registry: Arc<Registry>,
17246        snapshot: Arc<Mutex<SupervisorSnapshot>>,
17247        terminal_ring: Arc<Mutex<TerminalRing>>,
17248        spawn_events: SpawnEventFeed,
17249    }
17250
17251    fn fixture(label: &str, module_id: &str) -> Fixture {
17252        let dir = ScratchDir::new(label);
17253        let pid_file = dir.join("grandchild.pid");
17254        let supervisor = Supervisor::new_for_test(
17255            Arc::new(Registry::default()),
17256            RestartPolicy::new(3, Duration::ZERO),
17257        );
17258        let runtime = supervisor.runtime_config();
17259        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17260        let spec = ModuleSpec {
17261            module_id: module_id.to_string(),
17262            program: stub_path(),
17263            // Zero args deliberately: a `--subc` argument would make the stub dial
17264            // a daemon that is not there, and the failure would land in the same
17265            // stderr ring this fixture exists to keep quiet.
17266            args: Vec::new(),
17267            env: vec![
17268                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17269                (
17270                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17271                    pid_file.display().to_string(),
17272                ),
17273            ],
17274            reserved: false,
17275            reserved_prefixes: Vec::new(),
17276            protocol: ModuleProtocol::Subc,
17277            overlap: Default::default(),
17278        };
17279        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17280            .expect("spawn the supervised fixture");
17281        let grandchild = read_grandchild_pid(&pid_file);
17282        Fixture {
17283            _dir: dir,
17284            module_id: module_id.to_string(),
17285            grandchild,
17286            child: Some(child),
17287            registry: Arc::new(Registry::default()),
17288            snapshot,
17289            terminal_ring: Arc::clone(&runtime.terminal_ring),
17290            spawn_events: SpawnEventFeed::default(),
17291        }
17292    }
17293
17294    impl Fixture {
17295        /// Drain through the supervisor's own teardown path.
17296        async fn drain(&mut self) {
17297            let child = self
17298                .child
17299                .take()
17300                .expect("the fixture child is still present");
17301            drain_child_to_state(
17302                &self.module_id,
17303                ModuleProtocol::Subc,
17304                // No forwarding table in this fixture, so nothing reaches the
17305                // child over a connection.
17306                StopNotice::NotSent,
17307                &self.registry,
17308                None,
17309                &self.snapshot,
17310                &self.terminal_ring,
17311                &self.spawn_events,
17312                child,
17313                Duration::from_millis(500),
17314                ModuleState::Stopped,
17315                Some(false),
17316                &FailureContext::operator(),
17317            )
17318            .await
17319            .expect("drain the supervised fixture");
17320        }
17321    }
17322
17323    /// Teardown reaps the grandchild, not merely the direct child.
17324    ///
17325    /// This is the assertion the change exists for. Before containment the
17326    /// grandchild survived: it is a separate process, and `start_kill` is
17327    /// `TerminateProcess` scoped to one pid.
17328    #[tokio::test]
17329    async fn teardown_reaps_the_grandchild() {
17330        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17331        let grandchild = fixture.grandchild;
17332
17333        assert!(
17334            subc_jobobject::process_exists(grandchild),
17335            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17336        );
17337
17338        fixture.drain().await;
17339
17340        assert!(
17341            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17342            "grandchild {grandchild} outlived module teardown: the tree was not contained"
17343        );
17344    }
17345
17346    /// The mutation control: with containment withheld, the grandchild survives
17347    /// the same kill.
17348    ///
17349    /// This is the defect reproduction from #109 — a direct-child kill reaches
17350    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
17351    /// supervisor because `spawn_and_mark_running` now always contains on
17352    /// Windows, which is the point: there is no longer a path that spawns
17353    /// uncontained, so the control has to construct one.
17354    ///
17355    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
17356    /// grandchild ever dies here, that test is passing for a reason unrelated to
17357    /// the job object and the containment claim is unproven.
17358    #[test]
17359    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17360        let dir = ScratchDir::new("teardown-uncontained");
17361        let pid_file = dir.join("grandchild.pid");
17362        let mut child = std::process::Command::new(stub_path())
17363            .env("FAKE_AFT_NEVER_CONNECT", "1")
17364            .env(
17365                "FAKE_AFT_GRANDCHILD_PID_FILE",
17366                pid_file.display().to_string(),
17367            )
17368            .stdin(std::process::Stdio::null())
17369            .stdout(std::process::Stdio::null())
17370            .stderr(std::process::Stdio::null())
17371            .spawn()
17372            .expect("spawn the uncontained fixture");
17373        let grandchild = read_grandchild_pid(&pid_file);
17374
17375        // Exactly what the pre-fix teardown did: kill the direct child.
17376        child.kill().expect("kill the direct child");
17377        let _ = child.wait();
17378
17379        assert!(
17380            subc_jobobject::process_exists(grandchild),
17381            "grandchild {grandchild} died with the direct child, so this control no longer \
17382             distinguishes contained from uncontained teardown and the regression test is \
17383             passing vacuously"
17384        );
17385
17386        // The orphan this control demonstrates is the leak the fix prevents, so
17387        // the control must not leave one behind.
17388        kill_tree(grandchild);
17389    }
17390
17391    /// Crash durability: closing the containment handle reaps the tree with no
17392    /// teardown code running at all.
17393    ///
17394    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
17395    /// call anything — and it is why containment is a kernel property of the
17396    /// handle rather than a step in the drain. Discovered by getting the
17397    /// mutation control wrong: clearing `job` to "disable" containment instead
17398    /// killed the tree, which is the guarantee, not a mistake.
17399    #[tokio::test]
17400    async fn dropping_containment_reaps_the_grandchild() {
17401        let mut fixture = fixture("drop-containment", "tree-drop");
17402        let grandchild = fixture.grandchild;
17403
17404        assert!(subc_jobobject::process_exists(grandchild));
17405
17406        // No `drain` call, no kill: dropping the handle is the entire mechanism.
17407        fixture.child.as_mut().expect("child present").job = None;
17408
17409        assert!(
17410            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17411            "grandchild {grandchild} survived the containment handle closing, so a daemon \
17412             crash would leave the tree behind"
17413        );
17414    }
17415
17416    /// Kill a pid and its tree, then confirm it is gone.
17417    fn kill_tree(pid: u32) {
17418        let _ = std::process::Command::new("taskkill.exe")
17419            .args(["/PID", &pid.to_string(), "/T", "/F"])
17420            .stdin(std::process::Stdio::null())
17421            .stdout(std::process::Stdio::null())
17422            .stderr(std::process::Stdio::null())
17423            .status();
17424        assert!(
17425            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17426            "could not clean up grandchild {pid}"
17427        );
17428    }
17429}
17430
17431#[cfg(test)]
17432mod privacy_trampoline_configuration_tests {
17433    #[tokio::test]
17434    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17435    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17436        #[cfg(target_os = "macos")]
17437        {
17438            let supervisor = super::Supervisor::new(
17439                std::sync::Arc::new(crate::Registry::default()),
17440                super::RestartPolicy::default(),
17441            );
17442            let error = supervisor.spawn(spec()).unwrap_err();
17443            assert!(
17444                error
17445                    .to_string()
17446                    .contains("no privacy trampoline configured"),
17447                "{error}"
17448            );
17449        }
17450    }
17451
17452    #[tokio::test]
17453    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17454    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17455        #[cfg(target_os = "macos")]
17456        {
17457            let supervisor = super::Supervisor::new(
17458                std::sync::Arc::new(crate::Registry::default()),
17459                super::RestartPolicy::default(),
17460            )
17461            .with_privacy_trampoline(std::env::current_exe().unwrap());
17462            let error = supervisor.spawn(spec()).unwrap_err();
17463            assert!(
17464                error
17465                    .to_string()
17466                    .contains("binary does not implement the privacy trampoline protocol"),
17467                "{error}"
17468            );
17469        }
17470    }
17471
17472    #[cfg(target_os = "macos")]
17473    fn spec() -> super::ModuleSpec {
17474        super::ModuleSpec {
17475            module_id: "privacy-configuration".into(),
17476            program: "/bin/sleep".into(),
17477            args: vec!["30".into()],
17478            env: vec![],
17479            reserved: false,
17480            reserved_prefixes: vec![],
17481            protocol: subc_control::ModuleProtocol::None,
17482            overlap: super::ModuleOverlap::Exclusive,
17483        }
17484    }
17485}
17486
17487#[cfg(test)]
17488mod privacy_exec_boundary_tests {
17489    #[cfg(target_os = "macos")]
17490    use super::*;
17491    #[cfg(target_os = "macos")]
17492    use std::{
17493        io::{Read, Write},
17494        net::{TcpListener, TcpStream},
17495    };
17496
17497    /// Unit-test-only pause at the actual early image read, not at a later
17498    /// status read. Production supervisors never inspect this environment key.
17499    #[cfg(target_os = "macos")]
17500    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17501        if let Some((_, path)) = spec
17502            .env
17503            .iter()
17504            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17505        {
17506            let mut barrier = TcpStream::connect(path).unwrap();
17507            barrier
17508                .set_read_timeout(Some(Duration::from_secs(30)))
17509                .unwrap();
17510            barrier.write_all(&pid.to_ne_bytes()).unwrap();
17511            let mut release = [0];
17512            barrier.read_exact(&mut release).unwrap();
17513            assert_eq!(&release, b"X");
17514        }
17515    }
17516
17517    #[cfg(target_os = "macos")]
17518    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17519        ModuleSpec {
17520            module_id: "privacy-boundary".into(),
17521            program: program.into(),
17522            args: args.iter().map(|arg| (*arg).into()).collect(),
17523            env: vec![],
17524            reserved: false,
17525            reserved_prefixes: vec![],
17526            protocol: ModuleProtocol::None,
17527            overlap: ModuleOverlap::Exclusive,
17528        }
17529    }
17530
17531    #[cfg(target_os = "macos")]
17532    async fn accept(listener: TcpListener) -> TcpStream {
17533        // Socket readiness, not elapsed time, establishes both pause points.
17534        let listener = tokio::net::TcpListener::from_std({
17535            listener.set_nonblocking(true).unwrap();
17536            listener
17537        })
17538        .unwrap();
17539        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17540            .await
17541            .unwrap()
17542            .unwrap();
17543        let stream = stream.into_std().unwrap();
17544        stream.set_nonblocking(false).unwrap();
17545        stream
17546            .set_read_timeout(Some(Duration::from_secs(30)))
17547            .unwrap();
17548        stream
17549    }
17550
17551    #[tokio::test]
17552    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17553    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17554        #[cfg(target_os = "macos")]
17555        {
17556            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17557            // Loopback sockets also work when the replay adapter's TMPDIR is
17558            // longer than Darwin's Unix-domain socket path limit.
17559            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17560            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17561            let record = root.join("live-children.json");
17562            let supervisor =
17563                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17564                    .with_live_children_record(&record);
17565            let runtime = supervisor.runtime_config();
17566            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17567            let mut spec = spec("/bin/sleep", &["30"]);
17568            spec.env = vec![
17569                (
17570                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17571                    exec_listener.local_addr().unwrap().to_string(),
17572                ),
17573                (
17574                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17575                    sample_listener.local_addr().unwrap().to_string(),
17576                ),
17577            ];
17578            let spawn = tokio::task::spawn_blocking(move || {
17579                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17580            });
17581            let mut sample = accept(sample_listener).await;
17582            let mut pid = [0; 4];
17583            sample.read_exact(&mut pid).unwrap();
17584            let pid = u32::from_ne_bytes(pid);
17585            let mut exec = accept(exec_listener).await;
17586            let mut ready = [0];
17587            exec.read_exact(&mut ready).unwrap();
17588            assert_eq!(&ready, b"R");
17589            // The early read is guaranteed to see a real, non-null trampoline
17590            // image: the fixture has reached its barrier and cannot exec yet.
17591            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17592            assert_eq!(
17593                observe_spawned_image(pid).unwrap().executable,
17594                Some(trampoline)
17595            );
17596            sample.write_all(b"X").unwrap();
17597            let mut child = spawn.await.unwrap();
17598            let early = crate::live_children::read_record(&record).unwrap();
17599            assert_eq!(early.len(), 1);
17600            assert_eq!(early[0].pid, pid);
17601            assert_eq!(
17602                early[0].executable, None,
17603                "unconfirmed trampoline image entered the roster"
17604            );
17605            assert!(child.report_ready.get().is_none());
17606            // The barrier's duration is unrelated to the production five-second
17607            // exec budget. Start the test's confirmation budget upon release.
17608            child.privacy_exec.as_mut().unwrap().deadline =
17609                tokio::time::Instant::now() + Duration::from_secs(30);
17610            exec.write_all(b"X").unwrap();
17611            child.confirm_privacy_exec().await;
17612            assert_eq!(child.spawn_failure, None);
17613            assert!(child.report_ready.get().is_some());
17614            let confirmed = crate::live_children::read_record(&record).unwrap();
17615            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17616            assert_ne!(module, trampoline);
17617            assert_eq!(confirmed[0].executable, Some(module.into()));
17618            child.start_kill().unwrap();
17619            child.wait().await.unwrap();
17620            child.release_roster();
17621        }
17622    }
17623
17624    #[tokio::test]
17625    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17626    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17627        #[cfg(target_os = "macos")]
17628        {
17629            let registry = Arc::new(Registry::default());
17630            let policy = RestartPolicy::new(0, Duration::ZERO);
17631            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
17632            let runtime = supervisor.runtime_config();
17633            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17634            let spec = spec("/bin/sh", &["-c", "exit 121"]);
17635            // Drive spawn and confirmation separately instead of starting the
17636            // monitor. WNOWAIT observes a real exit without consuming its status,
17637            // so confirmation's first try_wait must take the already-exited arm.
17638            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17639            let pid = child.pid;
17640            tokio::task::spawn_blocking(move || {
17641                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17642            })
17643            .await
17644            .unwrap()
17645            .unwrap();
17646            child.privacy_exec.as_mut().unwrap().deadline =
17647                tokio::time::Instant::now() + Duration::from_secs(30);
17648            let status = child.wait().await.unwrap();
17649            assert_eq!(status.code(), Some(121));
17650            assert!(child.privacy_exec.is_none());
17651            assert!(
17652                child.report_ready.get().is_none(),
17653                "an exited module must not publish a live pid"
17654            );
17655            let report = classify_reaped_child_exit(&snapshot, &child, &status);
17656            on_child_exit(
17657                &spec,
17658                policy,
17659                &registry,
17660                &snapshot,
17661                &runtime.terminal_ring,
17662                &runtime.spawn_events,
17663                &runtime.child_roster,
17664                report,
17665                true,
17666            )
17667            .await;
17668            let state = lock_snapshot(&snapshot).unwrap();
17669            assert_eq!(state.state, ModuleState::Failed);
17670            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17671            assert_eq!(state.reported_pid(), None);
17672            drop(state);
17673            let history = runtime.terminal_ring.lock().unwrap().snapshot();
17674            assert_eq!(history.entries.len(), 1);
17675            let terminal = &history.entries[0];
17676            assert_eq!(terminal.exit_code, Some(121));
17677            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17678            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17679            assert_eq!(
17680                terminal.disposition_detail.as_deref(),
17681                Some(policy.budget_exhausted_detail().as_str()),
17682                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17683            );
17684            assert_eq!(child.spawn_failure, None);
17685            child.release_roster();
17686        }
17687    }
17688}
17689
17690/// The daemon's real spawn path hands a subc-wire child its launch nonce on
17691/// descriptor 3, without an environment copy. The shell records the nonce
17692/// and its environment after exec so these tests observe the real handover.
17693#[cfg(all(test, unix))]
17694mod launch_nonce_descriptor_tests {
17695    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17696    use crate::stderr_tail::{StderrRing, StderrTailConfig};
17697    use cortexkit_test_support::ScratchDir;
17698    use std::{
17699        path::PathBuf,
17700        sync::{Arc, Mutex},
17701        time::{Duration, Instant},
17702    };
17703
17704    async fn probe(role: super::SpawnRole) {
17705        let scratch = ScratchDir::new("launch-nonce-descriptor");
17706        let fd_copy = scratch.join("from-descriptor");
17707        let env_copy = scratch.join("environment");
17708        let script = format!(
17709            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17710            fd = fd_copy.display(), env = env_copy.display(),
17711        );
17712        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17713        let spec = ModuleSpec {
17714            module_id: "nonce-descriptor-probe".to_string(),
17715            program: PathBuf::from("/bin/sh"),
17716            args: vec!["-c".to_string(), script],
17717            env: vec![
17718                xdg("XDG_DATA_HOME"),
17719                xdg("XDG_RUNTIME_DIR"),
17720                xdg("XDG_CONFIG_HOME"),
17721                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17722            ],
17723            reserved: true,
17724            reserved_prefixes: Vec::new(),
17725            protocol: ModuleProtocol::Subc,
17726            overlap: Default::default(),
17727        };
17728        let handle = SupervisorHandle::new();
17729        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17730        let roster = ChildRoster::default();
17731        #[cfg(target_os = "macos")]
17732        {
17733            let path = super::test_privacy_trampoline();
17734            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17735        }
17736        let child = super::spawn_child_in_slot(
17737            &spec,
17738            None,
17739            Some(&handle),
17740            &ring,
17741            None,
17742            &roster,
17743            #[cfg(target_os = "linux")]
17744            None,
17745            role,
17746            matches!(role, super::SpawnRole::SwapCandidate),
17747        )
17748        .expect("spawn probe");
17749        let deadline = Instant::now() + Duration::from_secs(10);
17750        while !(fd_copy.exists() && env_copy.exists()) {
17751            assert!(Instant::now() < deadline, "probe never wrote its copies");
17752            tokio::time::sleep(Duration::from_millis(20)).await;
17753        }
17754        let nonce = std::fs::read_to_string(fd_copy).unwrap();
17755        assert!(!nonce.is_empty());
17756        let environment = std::fs::read_to_string(env_copy).unwrap();
17757        assert!(environment
17758            .lines()
17759            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17760        let copy = environment
17761            .lines()
17762            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17763        assert_eq!(
17764            copy, None,
17765            "Unix children must never receive the environment nonce"
17766        );
17767        if matches!(role, super::SpawnRole::Plain) {
17768            assert_eq!(
17769                handle.spawn_nonce(&spec.module_id).as_deref(),
17770                Some(nonce.as_str())
17771            );
17772        }
17773        drop(child);
17774    }
17775
17776    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17777    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17778        probe(super::SpawnRole::Plain).await;
17779    }
17780
17781    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17782    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17783        probe(super::SpawnRole::SwapCandidate).await;
17784    }
17785}
17786
17787#[cfg(all(test, target_os = "linux"))]
17788#[path = "supervise/cgroup_reap_tests.rs"]
17789mod cgroup_reap_tests;
17790
17791#[cfg(all(test, target_os = "linux"))]
17792mod cgroup_containment_tests {
17793    use super::*;
17794    use cortexkit_test_support::ScratchDir;
17795
17796    #[tokio::test]
17797    async fn linux_subtree_memory_covers_module_children() {
17798        let dir = ScratchDir::new("subtree-memory");
17799        let root = PathBuf::from(format!(
17800            "/sys/fs/cgroup/subc-memory-test-{}-{}",
17801            std::process::id(),
17802            unix_ms_now()
17803        ));
17804        if let Err(error) = std::fs::create_dir(&root) {
17805            assert!(
17806                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17807                "required cgroup test cannot execute: {error}"
17808            );
17809            eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
17810            return;
17811        }
17812        let placement = subc_cgroup::prepare_at(&root)
17813            .unwrap()
17814            .expect("delegated cgroup");
17815        let supervisor = Supervisor::new_for_test(
17816            Arc::new(Registry::default()),
17817            RestartPolicy::new(3, Duration::ZERO),
17818        )
17819        .with_cgroup_placement(Some(placement));
17820        let ready = dir.join("ready");
17821        let module = supervisor
17822            .spawn(ModuleSpec {
17823                module_id: "memory-tree".into(),
17824                program: PathBuf::from("/bin/sh"),
17825                args: vec![
17826                    "-c".into(),
17827                    "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
17828                    "fixture".into(),
17829                    ready.display().to_string(),
17830                ],
17831                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17832                    .into_iter()
17833                    .map(|key| (key.into(), dir.display().to_string()))
17834                    .collect(),
17835                reserved: false,
17836                reserved_prefixes: Vec::new(),
17837                protocol: ModuleProtocol::None,
17838                overlap: Default::default(),
17839            })
17840            .unwrap();
17841        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17842        while !ready.exists() {
17843            assert!(
17844                tokio::time::Instant::now() < deadline,
17845                "children never started"
17846            );
17847            tokio::time::sleep(Duration::from_millis(10)).await;
17848        }
17849        let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
17850        let reading = &readings[0];
17851        assert!(
17852            reading.process_count.is_some_and(|count| count >= 4),
17853            "{reading:?}"
17854        );
17855        assert!(
17856            reading.memory_bytes.is_some_and(|bytes| bytes > 0),
17857            "{reading:?}"
17858        );
17859        module.stop().await.unwrap();
17860        drop(module);
17861        std::fs::remove_dir(root.join("subc-modules")).unwrap();
17862        std::fs::remove_dir(&root).unwrap();
17863        eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
17864    }
17865
17866    fn running(pid: u32) -> bool {
17867        // An orphan can remain a zombie until the container init reaps it.
17868        std::fs::read_to_string(format!("/proc/{pid}/stat"))
17869            .ok()
17870            .and_then(|stat| {
17871                stat.rsplit_once(") ")
17872                    .map(|(_, rest)| rest.starts_with('Z'))
17873            })
17874            .is_some_and(|zombie| !zombie)
17875    }
17876
17877    #[tokio::test]
17878    async fn linux_teardown_reaps_the_grandchild() {
17879        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17880    }
17881
17882    #[tokio::test]
17883    async fn linux_shutdown_straggler_reaps_the_grandchild() {
17884        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17885    }
17886
17887    async fn teardown_tree(test_name: &str, shutdown: bool) {
17888        let dir = ScratchDir::new(test_name);
17889        let root = PathBuf::from(format!(
17890            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17891            std::process::id(),
17892            unix_ms_now()
17893        ));
17894        if let Err(error) = std::fs::create_dir(&root) {
17895            assert!(
17896                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17897                "required cgroup test cannot execute: {error}"
17898            );
17899            eprintln!(
17900                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17901                root.display()
17902            );
17903            return;
17904        }
17905        let placement = subc_cgroup::prepare_at(&root)
17906            .expect("prepare isolated kernel cgroup")
17907            .expect("isolated cgroup is delegated");
17908        let module_id = "tree-teardown";
17909        let module = placement
17910            .module_path(module_id)
17911            .expect("create isolated module cgroup");
17912        if !module.join("cgroup.kill").exists() {
17913            std::fs::remove_dir(&module).unwrap();
17914            std::fs::remove_dir(root.join("subc-modules")).unwrap();
17915            std::fs::remove_dir(&root).unwrap();
17916            assert!(
17917                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17918                "required cgroup.kill interface unavailable"
17919            );
17920            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17921            return;
17922        }
17923        let supervisor = Supervisor::new_for_test(
17924            Arc::new(Registry::default()),
17925            RestartPolicy::new(3, Duration::ZERO),
17926        )
17927        .with_cgroup_placement(Some(placement));
17928        let mut runtime = supervisor.runtime_config();
17929        runtime.child_roster = runtime
17930            .child_roster
17931            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17932        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17933        let pid_file = dir.join("grandchild.pid");
17934        let spec = ModuleSpec {
17935            module_id: module_id.to_string(),
17936            program: PathBuf::from("/bin/sh"),
17937            args: vec![
17938                "-c".into(),
17939                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17940                "fixture".into(),
17941                pid_file.display().to_string(),
17942            ],
17943            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17944                .into_iter()
17945                .map(|key| (key.to_string(), dir.display().to_string()))
17946                .collect(),
17947            reserved: false,
17948            reserved_prefixes: Vec::new(),
17949            protocol: ModuleProtocol::None,
17950            overlap: Default::default(),
17951        };
17952        let child =
17953            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17954        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17955        let grandchild: u32 = loop {
17956            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17957                if let Ok(pid) = contents.trim().parse() {
17958                    break pid;
17959                }
17960            }
17961            assert!(
17962                tokio::time::Instant::now() < deadline,
17963                "grandchild pid was not recorded"
17964            );
17965            tokio::time::sleep(Duration::from_millis(10)).await;
17966        };
17967        assert!(
17968            running(grandchild),
17969            "grandchild must be alive before teardown"
17970        );
17971        if shutdown {
17972            let mut child = child;
17973            crate::child_roster::end_children_for_daemon_shutdown(
17974                &runtime.child_roster,
17975                false,
17976                std::future::pending(),
17977            )
17978            .await;
17979            child.wait().await.expect("reap shutdown straggler");
17980        } else {
17981            drain_child_to_state(
17982                module_id,
17983                ModuleProtocol::None,
17984                StopNotice::NotSent,
17985                &Registry::default(),
17986                None,
17987                &snapshot,
17988                &runtime.terminal_ring,
17989                &SpawnEventFeed::default(),
17990                child,
17991                Duration::from_millis(100),
17992                ModuleState::Stopped,
17993                Some(false),
17994                &FailureContext::operator(),
17995            )
17996            .await
17997            .expect("real supervisor teardown");
17998        }
17999        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18000        while running(grandchild) && tokio::time::Instant::now() < deadline {
18001            tokio::time::sleep(Duration::from_millis(10)).await;
18002        }
18003        let survived = running(grandchild);
18004        // Kill a surviving grandchild so a failed test does not leave it behind.
18005        if survived {
18006            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18007            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18008            tokio::time::sleep(Duration::from_millis(100)).await;
18009        }
18010        if module.exists() {
18011            std::fs::remove_dir(&module).expect("remove empty module cgroup");
18012        }
18013        std::fs::remove_dir(root.join("subc-modules")).unwrap();
18014        std::fs::remove_dir(&root).unwrap();
18015        assert!(
18016            !survived,
18017            "grandchild {grandchild} outlived module teardown"
18018        );
18019        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18020    }
18021}
18022
18023/// A slow live-children record write on the restart path must not stop the
18024/// daemon from answering the replacement module's authentication handshake.
18025#[cfg(all(test, unix))]
18026mod record_write_stall_tests {
18027    use super::*;
18028    use std::sync::atomic::{AtomicBool, Ordering};
18029    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18030
18031    /// `subc-client-rs` gives the whole handshake this long before the module
18032    /// exits with `Timeout { stage: ServerProof }`.
18033    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18034    /// How long the replacement's admission write is held: longer than the
18035    /// module's deadline, standing in for the delay a busy disk can add
18036    /// across the two syncs in that write.
18037    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18038
18039    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
18040    /// On macOS every module is launched through the privacy trampoline, and
18041    /// a module that is not a `#!` script is admitted only if its running
18042    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
18043    /// the shell soon replaces itself with `/bin/sleep`, which does not match
18044    /// the configured `/bin/sh`. The image is read only after the spawn's
18045    /// live-children record write, and here that write lasted long enough for
18046    /// the shell to have exec'd every time, so each process was refused and
18047    /// killed right after spawn, and the supervisor crash-looped it with
18048    /// backoff until its restart budget ran out. The restart this test
18049    /// measures would then never produce a module at all.
18050    fn sleeping_spec() -> ModuleSpec {
18051        ModuleSpec {
18052            module_id: "record-write-stall".into(),
18053            program: "/bin/sleep".into(),
18054            args: vec!["600".into()],
18055            env: vec![],
18056            reserved: false,
18057            reserved_prefixes: vec![],
18058            protocol: ModuleProtocol::None,
18059            overlap: Default::default(),
18060        }
18061    }
18062
18063    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18064    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18065        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18066        let supervisor =
18067            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18068                .with_live_children_record(home.join("live-children.json"));
18069        let module = supervisor.spawn(sleeping_spec()).unwrap();
18070        // `reported_pid` names a process only once it is confirmed as the
18071        // module (on macOS, once the trampoline's image check has passed), so
18072        // a process the daemon is about to refuse and kill never counts as
18073        // the running module, here or in the final wait below.
18074        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18075            loop {
18076                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18077                    .unwrap()
18078                    .reported_pid()
18079                {
18080                    return pid;
18081                }
18082                tokio::time::sleep(Duration::from_millis(10)).await;
18083            }
18084        })
18085        .await
18086        .expect("the first process was admitted as the module");
18087
18088        // The daemon's listener, served by this runtime's workers like the
18089        // supervisor's own tasks.
18090        let key = vec![0x42; 32];
18091        let daemon_id = [0x24; 16];
18092        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18093        let port = listener.local_addr().unwrap().port();
18094        let server = tokio::spawn(crate::server::serve_listener(
18095            listener,
18096            Arc::new(crate::router::Router::with_default_self_handler()),
18097            crate::server::ServerAuth::with_limits(
18098                key.clone(),
18099                daemon_id,
18100                "test",
18101                crate::server::DEFAULT_AUTH_DEADLINE,
18102                4,
18103            ),
18104        ));
18105
18106        // Hold the write that admits the replacement, once, and say when it
18107        // has started.
18108        let (held_tx, held_rx) = std::sync::mpsc::channel();
18109        let armed = AtomicBool::new(true);
18110        supervisor
18111            .child_roster
18112            .set_record_write_hook_for_test(move |live| {
18113                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18114                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18115                    let _ = held_tx.send(());
18116                    std::thread::sleep(HELD_RECORD_WRITE);
18117                }
18118            });
18119
18120        // The replacement module, as a separate process would be: its own
18121        // thread and runtime, so only the daemon's runtime can be stalled.
18122        let module_process = std::thread::spawn(move || {
18123            held_rx
18124                .recv_timeout(Duration::from_secs(30))
18125                .expect("the restart reached the replacement's record write");
18126            let conn = ConnectionInfo {
18127                schema: SCHEMA_VERSION,
18128                wire_version: None,
18129                endpoints: vec![Endpoint {
18130                    host: "127.0.0.1".to_owned(),
18131                    port,
18132                }],
18133                key,
18134                daemon_id,
18135                pid: std::process::id(),
18136                daemon_ver: "test".to_owned(),
18137            };
18138            tokio::runtime::Builder::new_current_thread()
18139                .enable_all()
18140                .build()
18141                .unwrap()
18142                .block_on(async {
18143                    let started = Instant::now();
18144                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18145                        .await
18146                        .unwrap();
18147                    let result =
18148                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18149                    (result, started.elapsed())
18150                })
18151        });
18152
18153        module.restart(Some(0)).await.unwrap();
18154        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18155            .await
18156            .unwrap();
18157        assert!(
18158            auth.is_ok(),
18159            "the daemon must answer a module's handshake while a restart waits on the \
18160             live-children record write; got {auth:?} after {elapsed:?}"
18161        );
18162
18163        tokio::time::timeout(Duration::from_secs(20), async {
18164            while lock_snapshot(&module.inner.snapshot)
18165                .unwrap()
18166                .reported_pid()
18167                .is_none_or(|pid| pid == old_pid)
18168            {
18169                tokio::time::sleep(Duration::from_millis(10)).await;
18170            }
18171        })
18172        .await
18173        .expect("the restart published its replacement");
18174        module.stop().await.unwrap();
18175        server.abort();
18176    }
18177}