Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62/// Command-line flag used by supervised modules to find subc.
63///
64/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
65/// The path points at the TCP+key connection file; it is not an ambient signal and
66/// is never inherited by standalone children.
67pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
73/// to contain a real crash loop (which respawns in seconds) and short enough
74/// that unrelated crashes hours apart never accumulate into a permanent stop.
75const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76/// How long a drain waits for already-dispatched requests to finalize before
77/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
78/// not RPC-scale: the original 2s value silently cut nearly every real tool
79/// call at the fence, making the wait-for-finalize design decorative for the
80/// workloads it existed for. Quiescence short-circuits, so an idle module
81/// restarts immediately regardless of this value; the budget is spent only
82/// when a genuine in-flight request is worth finishing. Per-module override:
83/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
84/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
85/// where a stuck request will never settle).
86pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89/// How long a drain waits after one busy-gauge probe completes before it asks
90/// again.
91///
92/// A drain re-probes on this short interval of its own rather than on the
93/// module's health cadence. The default cadence (30 s) equals the default
94/// drain ceiling, so a cadence-paced drain that found the module busy at its
95/// first probe never asked a second time: it waited out the whole ceiling and
96/// forced GOODBYE even after the gauges had read zero. The interval is counted
97/// from the moment the previous probe finished, and probes run one at a time,
98/// so a slow module is never asked more often than it can answer. Periodic
99/// health checks outside a drain keep using the configured cadence.
100const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101/// How long a restart waits for the exited process's output readers.
102///
103/// The restart does not depend on either output reader finishing. A stderr
104/// reader still running at this bound is left running, and its later output goes
105/// into the exited process's own section of the stderr ring (see
106/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
107/// bound no longer decides whether a crash's last lines are kept: under load
108/// the reader may simply not have been scheduled yet, and cutting it there
109/// discarded exactly the lines that explained the crash.
110///
111/// What the bound still decides is when the tail starts reporting
112/// `Incomplete`: a pipe open past it usually means a descendant of the exited
113/// process still holds it, and the tail cannot claim to be whole until that
114/// pipe closes. That is also the only case in which the wait costs the restart
115/// anything, because a pipe with no other holder reaches EOF when the process
116/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
117/// `Captured` at EOF with nothing lost.
118///
119/// The stdout reader carries no ring, only the capture file. It also continues
120/// to EOF after this bound: child exit closes the writer but does not mean the
121/// reader has consumed all buffered bytes. Late output from either pipe can
122/// therefore reach the shared capture file after the successor starts.
123const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
125pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
128/// dropped because the subscriber stopped draining its frames. The detail's
129/// `first_undelivered_cursor` names the first event it did not receive; the
130/// client resubscribes from the last cursor it did receive.
131pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136    deadline: tokio::time::Instant,
137    expected: Option<subc_os::FileIdentity>,
138    trampoline: Option<subc_os::FileIdentity>,
139    script: bool,
140    module_id: String,
141    #[cfg(test)]
142    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
148    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
149    let path = std::env::current_exe()
150        .unwrap()
151        .parent()
152        .unwrap()
153        .parent()
154        .unwrap()
155        .join("privacy-trampoline-fixture");
156    // Without the fixture every macOS spawn is refused, and the tests that
157    // spawn fail later as a module in state Failed, which names the wrong
158    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
159    assert!(
160        path.exists(),
161        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162         --bins --features test-support` or `cargo test -p subc-daemon` first",
163        path.display()
164    );
165    path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170    use std::io::Read;
171    let mut probe = std::process::Command::new(path)
172        .args(["__disclaim-exec", "--probe"])
173        .stdin(Stdio::null())
174        .stdout(Stdio::piped())
175        .stderr(Stdio::piped())
176        .spawn()
177        .map_err(|error| {
178            format!(
179                "privacy trampoline probe failed for {}: {error}",
180                path.display()
181            )
182        })?;
183    let deadline = std::time::Instant::now() + Duration::from_secs(5);
184    let status = loop {
185        match probe.try_wait() {
186            Ok(Some(status)) => break status,
187            Ok(None) if std::time::Instant::now() < deadline => {
188                std::thread::sleep(Duration::from_millis(5))
189            }
190            result => {
191                let _ = probe.kill();
192                let _ = probe.wait();
193                return Err(format!(
194                    "privacy trampoline probe failed or timed out for {}: {result:?}",
195                    path.display()
196                ));
197            }
198        }
199    };
200    let mut answer = String::new();
201    if let Some(stdout) = probe.stdout.take() {
202        let _ = stdout.take(256).read_to_string(&mut answer);
203    }
204    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205        return Ok(());
206    }
207    let mut diagnostic = String::new();
208    if let Some(stderr) = probe.stderr.take() {
209        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210    }
211    let cause = diagnostic
212        .trim()
213        .strip_prefix("ck-subc: own privacy identity refused: ")
214        .unwrap_or("binary does not implement the privacy trampoline protocol");
215    Err(format!(
216        "{cause}: probe of {} exited {status}",
217        path.display()
218    ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223    spec: &ModuleSpec,
224    roster: &ChildRoster,
225) -> Result<
226    (
227        Command,
228        Option<PrivacyExec>,
229        subc_os::privacy_identity::ExecAcknowledgement,
230    ),
231    SuperviseError,
232> {
233    let failure = |cause: String| {
234        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235        SuperviseError::Spawn {
236            program: spec.program.clone(),
237            source: io::Error::other(cause),
238            cgroup_path: None,
239        }
240    };
241    let trampoline = roster.privacy_trampoline().map_err(failure)?;
242    // Resolve PATH with the same environment the Command will receive. For
243    // scripts retain the existing orphan-identity rule: the kernel chooses
244    // the interpreter, and its observed image is the one recorded. Do not
245    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
246    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247        let path = spec
248            .env
249            .iter()
250            .find(|(key, _)| key == "PATH")
251            .map(|(_, value)| std::ffi::OsString::from(value))
252            .or_else(|| std::env::var_os("PATH"))
253            .unwrap_or_else(|| "/usr/bin:/bin".into());
254        std::env::split_paths(&path)
255            .map(|dir| dir.join(&spec.program))
256            .find(|path| path.is_file())
257            .unwrap_or_else(|| spec.program.clone())
258    } else {
259        spec.program.clone()
260    };
261    let expected = subc_os::file_identity(&program);
262    let trampoline_image = subc_os::file_identity(&trampoline);
263    let script = {
264        use std::io::Read;
265        let mut prefix = [0u8; 2];
266        std::fs::File::open(&program)
267            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268    };
269    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270        return Err(failure(
271            "privacy identity module executable is missing or is the trampoline itself".to_string(),
272        ));
273    }
274    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275        .map_err(|error| failure(error.to_string()))?;
276    let reader =
277        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278    let mut command = Command::new(&trampoline);
279    command
280        .arg("__disclaim-exec")
281        .arg(ack.fd().to_string())
282        .arg(&program);
283    ack.install(command.as_std_mut());
284    Ok((
285        command,
286        Some(PrivacyExec {
287            reader,
288            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289            expected,
290            trampoline: trampoline_image,
291            script,
292            module_id: spec.module_id.clone(),
293            #[cfg(test)]
294            confirmation_pause: spec
295                .env
296                .iter()
297                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298                .map(|(_, address)| (address.clone(), None)),
299        }),
300        ack,
301    ))
302}
303
304struct SupervisedChild {
305    child: Child,
306    exec: crate::activity::ExecOwner,
307    #[cfg(windows)]
308    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309    #[cfg(target_os = "macos")]
310    privacy_exec: Option<PrivacyExec>,
311    /// Set once this launch's exec acknowledgement confirms the module image.
312    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
313    /// `subc_os::privacy_identity`), which then replaces itself with the
314    /// module. The supervisor owns and can kill that pid from spawn, but
315    /// status readers report it only after this latch is set, so nothing
316    /// reports the trampoline's image as the module's.
317    #[cfg(target_os = "macos")]
318    report_ready: Arc<OnceLock<()>>,
319    /// Refusal before the module image was accepted, retained for terminal records.
320    spawn_failure: Option<String>,
321    /// The protocol this process was launched with. A reload can store a new
322    /// launch spec with a different protocol, but that takes effect only at the
323    /// next spawn, so this process keeps being handled by the protocol it
324    /// actually speaks.
325    protocol: ModuleProtocol,
326    /// This process's cgroup name: a bounded module/slot label followed by a
327    /// spawn suffix unique to this process (when cgroup placement is on). A
328    /// retired process in a slot may still be draining when a later one is
329    /// spawned into that slot, so the suffix keeps the later process out of
330    /// the retired one's cgroup, which is the domain a kill applies to.
331    #[cfg(target_os = "linux")]
332    module_id: String,
333    #[cfg(target_os = "linux")]
334    cgroup_placement: Option<subc_cgroup::Placement>,
335    /// The job that contains this child and every process it spawns (issue #109).
336    ///
337    /// Dropping this handle is what reaps a surviving tree when no supervisor
338    /// code runs — a daemon crash — because the job carries
339    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
340    ///
341    /// That limit is not crash-only, and the difference is worth knowing: a
342    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
343    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
344    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
345    /// module at once. Before this change they survived that, saw EOF on the
346    /// control socket, and ran their own teardown; Unix keeps that path
347    /// deliberately, so a module can seal a WAL or close a capture rather than
348    /// be killed mid-write. So this trades graceful teardown on every Windows
349    /// daemon stop for containment on a crash, which is the right way round
350    /// today: orphaned GPU workers are a reported, recurring problem, and the
351    /// modules that write most heavily do not run on Windows.
352    ///
353    /// The fix is a real Windows stop path — the daemon draining before it
354    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
355    /// reaches only what the drain left behind, which is what it should reach.
356    #[cfg(windows)]
357    job: Option<subc_jobobject::JobObject>,
358    stdout_pump: Option<JoinHandle<()>>,
359    stderr_pump: Option<StderrPump>,
360    stderr_ring: Arc<Mutex<StderrRing>>,
361    /// Conservative monotonic bound sampled before `command.spawn()`, not when
362    /// identity recording finishes or the spawn generation is published.
363    spawned_at: Instant,
364    spawned_at_ms: u64,
365    spawned_from: PathBuf,
366    launch_spec_digest: LaunchSpecDigest,
367    spawned_file_identity: Option<SpawnedFileIdentity>,
368    process_start_time: Option<u64>,
369    process_identity: Option<ProcessIdentity>,
370    pid: u32,
371    /// This process's entry in the daemon's child roster, released when the
372    /// process is reaped or this handle is dropped.
373    roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377    fn id(&self) -> Option<u32> {
378        Some(self.pid)
379    }
380
381    fn process_identity(&self) -> Option<ProcessIdentity> {
382        self.process_identity
383    }
384
385    async fn wait(&mut self) -> io::Result<ExitStatus> {
386        #[cfg(target_os = "macos")]
387        self.confirm_privacy_exec().await;
388        // The roster entry is NOT released here. A daemon shutdown waits for the
389        // roster to empty and then exits the process, so releasing at the reap
390        // let it exit before the exit handler wrote this child's terminal record
391        // (the stderr drain and snapshot update sit in between), and the
392        // shutdown's own `daemon_shutdown` record was intermittently lost. The
393        // caller releases it after recording the exit (`release_roster`), and
394        // dropping the handle releases it too.
395        let result = self.child.wait().await;
396        #[cfg(target_os = "linux")]
397        if result.is_ok() {
398            if let Some(placement) = self.cgroup_placement.as_ref() {
399                cleanup_reaped_cgroup(placement, &self.module_id).await;
400                // Keep ownership while awaiting kernel population changes: a
401                // drain timeout may cancel this wait and then escalate/reap.
402                self.cgroup_placement = None;
403            }
404        }
405        result
406    }
407
408    #[cfg(target_os = "macos")]
409    async fn confirm_privacy_exec(&mut self) {
410        let Some(pending) = &mut self.privacy_exec else {
411            return;
412        };
413        let result = tokio::time::timeout_at(pending.deadline, async {
414            let mut record = Vec::new();
415            loop {
416                let mut ready = pending.reader.readable().await?;
417                let read = ready.try_io(|reader| {
418                    use std::io::Read;
419                    let mut reader = reader.get_ref();
420                    let mut buffer = [0u8; 256];
421                    reader.read(&mut buffer).map(|count| (count, buffer))
422                });
423                match read {
424                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425                    Ok(Ok((count, buffer))) => {
426                        if record.len() + count > 1024 {
427                            return Err(io::Error::other(
428                                "privacy exec refusal record is too long",
429                            ));
430                        }
431                        record.extend_from_slice(&buffer[..count]);
432                    }
433                    Ok(Err(error)) => return Err(error),
434                    Err(_) => continue,
435                }
436            }
437        })
438        .await;
439        #[cfg(test)]
440        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441            use tokio::io::{AsyncReadExt, AsyncWriteExt};
442            if stream.is_none() {
443                *stream = Some(
444                    tokio::net::TcpStream::connect(address.as_str())
445                        .await
446                        .unwrap(),
447                );
448                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449                stream
450                    .as_mut()
451                    .unwrap()
452                    .write_all(&self.pid.to_ne_bytes())
453                    .await
454                    .unwrap();
455            }
456            let mut reply = [0];
457            stream
458                .as_mut()
459                .unwrap()
460                .read_exact(&mut reply)
461                .await
462                .unwrap();
463            pending.confirmation_pause = None;
464            if &reply == b"F" {
465                Ok(Err(io::Error::other("injected late exec refusal")))
466            } else {
467                assert_eq!(&reply, b"X");
468                result
469            }
470        } else {
471            result
472        };
473        // Keep the reader in self across await: select cancellation must not
474        // discard the handshake or reset its original five-second deadline.
475        let pending = self.privacy_exec.as_ref().expect("pending exec");
476        let cause = match result {
477            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478            Ok(Err(error)) => Some(format!(
479                "privacy identity exec acknowledgement failed: {error}"
480            )),
481            Ok(Ok(record)) if !record.is_empty() => Some(
482                std::str::from_utf8(&record)
483                    .ok()
484                    .and_then(|record| {
485                        record
486                            .trim()
487                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488                    })
489                    .filter(|cause| !cause.is_empty())
490                    .unwrap_or("invalid privacy exec refusal record")
491                    .to_string(),
492            ),
493            Ok(Ok(_)) => match self.child.try_wait() {
494                // Empty EOF is the exec acknowledgement. A real module may exit
495                // immediately, including with a reserved trampoline status; no
496                // image is admitted, and its ordinary exit contract stays intact.
497                Ok(Some(_status)) => None,
498                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499                Ok(None) => {
500                    let image = observe_spawned_image(self.pid);
501                    if let Some(image) = image.filter(|image| {
502                        image.executable.is_some()
503                            && image.executable != pending.trampoline
504                            && (image.executable == pending.expected || pending.script)
505                    }) {
506                        if let Err(error) = self
507                            .roster_guard
508                            .as_ref()
509                            .map(|guard| guard.confirm_executable(image))
510                            .transpose()
511                        {
512                            Some(format!(
513                                "could not durably record confirmed module image: {error}"
514                            ))
515                        } else {
516                            let _ = self.report_ready.set(());
517                            info!(module_id = %pending.module_id, pid = self.pid,
518                                "module spawned with own privacy identity (responsibility disclaimed)");
519                            None
520                        }
521                    } else if image.is_none()
522                        || image.is_some_and(|image| image.executable.is_none())
523                    {
524                        // A process can exit between try_wait and the kernel
525                        // image read. Empty EOF already acknowledged exec, so
526                        // preserve that module's ordinary exit rather than
527                        // mislabel a disappearing image as trampoline refusal.
528                        // Keep pending in self across await so cancellation does
529                        // not discard validation or reset its original deadline.
530                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531                            Ok(Ok(_status)) => None,
532                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534                        }
535                    } else {
536                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537                    }
538                }
539            },
540        };
541        let pending = self.privacy_exec.take().expect("pending exec");
542        // Exec confirmation describes launch success, not roster retirement.
543        // Keep even a reaped child recorded until its terminal history exists.
544        self.exec.0.finish(if cause.is_some() {
545            crate::activity::ExecState::Refused
546        } else {
547            crate::activity::ExecState::CommittedExec
548        });
549        if let Some(cause) = cause {
550            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551            self.spawn_failure = Some(cause);
552            // No image is admitted on failure. Reach the entire fresh process
553            // group, including a module which spawned a helper before refusal.
554            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556            }
557            let _ = self.child.start_kill();
558        }
559    }
560
561    /// Releases this child's daemon-shutdown roster entry once its exit has
562    /// been recorded. The pid is already reaped and free for reuse, so the
563    /// entry must not outlive the record any longer than that.
564    fn release_roster(&mut self) {
565        self.roster_guard = None;
566    }
567
568    /// Kill the child and, where containment is available, its process tree.
569    ///
570    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
571    /// helper process leaked the helper — the Synapse embedding module's CUDA
572    /// worker holds the GPU allocation, so the leak cost VRAM until the next
573    /// restart of something else. Terminating the job reaches grandchildren that
574    /// a tree walk cannot, including one whose parent has already exited and
575    /// been reparented away.
576    ///
577    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
578    /// direct-child kill still decides the outcome, so containment can never
579    /// change whether a module is reported as stopped.
580    fn start_kill(&mut self) -> io::Result<()> {
581        #[cfg(windows)]
582        if let Some(job) = &self.job {
583            if let Err(error) = job.terminate() {
584                debug!(
585                    error = %error,
586                    "job termination failed; the direct-child kill still owns the outcome"
587                );
588            }
589        }
590        #[cfg(target_os = "linux")]
591        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592        self.child.start_kill()
593    }
594
595    async fn drain_stderr(&mut self, module_id: &str) {
596        if let Some(pump) = self.stdout_pump.take() {
597            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598        }
599
600        let Some(pump) = self.stderr_pump.take() else {
601            return;
602        };
603        settle_stderr_pump(
604            module_id,
605            &self.stderr_ring,
606            pump,
607            STDERR_PUMP_DRAIN_TIMEOUT,
608        )
609        .await;
610    }
611}
612
613/// Bound the restart's wait, not the lifetime of the exited child's stdout
614/// reader. Slow pipe reads or capture writes must not turn the bound into lost
615/// output; dropping the handle lets the reader finish at EOF, like stderr.
616async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617    match timeout(bound, &mut pump).await {
618        Ok(Ok(())) => {}
619        Ok(Err(error)) => {
620            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621        }
622        Err(_) => {
623            drop(pump);
624            warn!(
625                module_id,
626                waited = ?bound,
627                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628            );
629        }
630    }
631}
632
633/// The reader task for one process's stderr, with the ring generation its
634/// lines are attributed to.
635struct StderrPump {
636    task: JoinHandle<()>,
637    generation: u64,
638}
639
640/// Retire an exited process's stderr reader and wait up to `bound` for it to
641/// reach EOF. A reader still running at the bound is detached, not stopped: it
642/// keeps filling the exited process's section of the ring until its pipe
643/// closes, and the tail reads `Incomplete` until then. See
644/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
645async fn settle_stderr_pump(
646    module_id: &str,
647    ring: &Arc<Mutex<StderrRing>>,
648    pump: StderrPump,
649    bound: Duration,
650) {
651    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652    let StderrPump {
653        mut task,
654        generation,
655    } = pump;
656    lock().retire_pump(generation);
657    match timeout(bound, &mut task).await {
658        Ok(Ok(())) => {}
659        Ok(Err(err)) => {
660            let mut ring = lock();
661            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662            ring.finish_pump(generation);
663            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664        }
665        Err(_) => {
666            // Dropping the handle detaches the task; it ends at EOF on its pipe.
667            drop(task);
668            lock().mark_pump_late(
669                generation,
670                format!(
671                    "stderr of the exited process had not reached EOF {bound:?} after it was \
672                     retired (a descendant may still hold the pipe open); lines it still \
673                     writes are kept in that process's section"
674                ),
675            );
676            warn!(
677                module_id,
678                waited = ?bound,
679                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680            );
681        }
682    }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687    EVENTS.get_or_init(|| {
688        let (sender, _receiver) = watch::channel(0);
689        sender
690    })
691}
692
693pub(crate) fn notify_registration_release() {
694    let events = registration_release_events();
695    let next_generation = (*events.borrow()).wrapping_add(1);
696    events.send_replace(next_generation);
697}
698
699/// How to launch one singleton module process.
700#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702    pub module_id: String,
703    pub program: PathBuf,
704    pub args: Vec<String>,
705    pub env: Vec<(String, String)>,
706    /// When true this is a reserved module: each spawn gets a fresh one-time launch
707    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
708    /// process can register this module_id (a security-boundary module like the
709    /// credential vault must not be impersonable while it is down/restarting).
710    pub reserved: bool,
711    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
712    /// Prefixes come from daemon config and must end in `:` before they reach the
713    /// supervisor; the owner module's current spawn nonce authorizes claims under
714    /// each prefix.
715    pub reserved_prefixes: Vec<String>,
716    /// The wire protocol this module speaks, as DECLARED in daemon config.
717    ///
718    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
719    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
720    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
721    /// and NO launch nonce, and a clean exit the daemon did not request is
722    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
723    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
724    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
725    /// because a process ignores an environment variable it does not read.
726    ///
727    /// The argument is the part that cannot be "harmless to a process that
728    /// ignores it": a stock binary exits on an unknown flag before it listens
729    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
730    /// first conformance run against this mode found it. The nonce is withheld
731    /// because a process that will never present it gains nothing from holding
732    /// it, and a secret in the environment of a process that does not need it is
733    /// a leak surface for no benefit.
734    pub protocol: ModuleProtocol,
735    /// Whether two processes of this module may run at once, which is what a
736    /// blue/green swap does for the length of its overlap. Declared in daemon
737    /// config because the daemon must be able to answer it while the module is
738    /// down, and so a module cannot talk itself into it after registering.
739    pub overlap: ModuleOverlap,
740}
741
742/// Only the configured launch inputs enter this value; spawned connection paths,
743/// nonces and inherited environment never do. Its bytes remain daemon-private.
744#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749        f.write_str("LaunchSpecDigest([redacted])")
750    }
751}
752
753impl LaunchSpecDigest {
754    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755        fn field(hash: &mut Sha256, bytes: &[u8]) {
756            hash.update((bytes.len() as u64).to_le_bytes());
757            hash.update(bytes);
758        }
759        let mut hash = Sha256::new();
760        if let Some(program) = spec.program.to_str() {
761            field(&mut hash, b"utf8-program");
762            field(&mut hash, program.as_bytes());
763        } else {
764            // The daemon config is UTF-8, but in-process callers may provide
765            // native paths: lossy conversion would collapse distinct programs.
766            #[cfg(unix)]
767            {
768                use std::os::unix::ffi::OsStrExt;
769                field(&mut hash, b"unix-program");
770                field(&mut hash, spec.program.as_os_str().as_bytes());
771            }
772            #[cfg(windows)]
773            {
774                use std::os::windows::ffi::OsStrExt;
775                field(&mut hash, b"windows-program");
776                let bytes = spec
777                    .program
778                    .as_os_str()
779                    .encode_wide()
780                    .flat_map(u16::to_le_bytes)
781                    .collect::<Vec<_>>();
782                field(&mut hash, &bytes);
783            }
784            #[cfg(not(any(unix, windows)))]
785            field(&mut hash, spec.program.to_string_lossy().as_bytes());
786        }
787        hash.update((spec.args.len() as u64).to_le_bytes());
788        for arg in &spec.args {
789            field(&mut hash, arg.as_bytes());
790        }
791        let mut env = spec.env.iter().collect::<Vec<_>>();
792        env.sort_unstable();
793        hash.update((env.len() as u64).to_le_bytes());
794        for (key, value) in env {
795            field(&mut hash, key.as_bytes());
796            field(&mut hash, value.as_bytes());
797        }
798        field(
799            &mut hash,
800            match spec.protocol {
801                ModuleProtocol::Subc => b"subc",
802                ModuleProtocol::None => b"none",
803            },
804        );
805        Self(hash.finalize().into())
806    }
807}
808
809/// Whether a module tolerates a second process of itself running alongside.
810///
811/// Most modules are single-writer on their store (a WAL, a capture log, a
812/// resident index behind a writer barrier), and two processes on one store
813/// corrupt it. So a swap, which overlaps the old and new process by design,
814/// is refused unless the module's config opts in with `overlap: "safe"`.
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817    /// Never run two processes of this module at once. The default.
818    #[default]
819    Exclusive,
820    /// The module has said a second process of itself is harmless for the
821    /// length of a swap.
822    ///
823    /// Declare it only if a second instance can run for a few seconds without
824    /// touching ANY single-writer store: every database, WAL, index, projector
825    /// and scheduled job the module owns. A lease on part of that state is not
826    /// enough. broca's session lease guards WAL appends while its run index, its
827    /// store projector and its archive fold timer (which unlinks live WAL files)
828    /// stay single-writer, so broca is exclusive despite holding a lease. The
829    /// refusal only fires after this has been decided, so the decision is the
830    /// check.
831    Safe,
832}
833
834impl ModuleOverlap {
835    pub fn as_str(self) -> &'static str {
836        match self {
837            Self::Exclusive => "exclusive",
838            Self::Safe => "safe",
839        }
840    }
841}
842
843/// Environment variable telling a spawned module which case it was started
844/// for, before it sends HELLO. Only a swap candidate carries it, as
845/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
846///
847/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
848/// longer because nobody waits on it, while a plain restart must flip ready
849/// quickly because callers see `module_warming` until it does. Absence means
850/// plain restart, the safe reading. The daemon trusts nothing about it; the
851/// candidate is proven by its launch nonce at HELLO.
852pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
854pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855/// How long a swap waits for its candidate to register and declare itself
856/// ready when the operator does not say. A module warming as a swap candidate
857/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
858/// daemon allows that plus time to start the process and send HELLO.
859pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861/// Bounded restart policy for crash exits.
862///
863/// `max_restarts` is the number of replacement processes allowed after the
864/// initial spawn WITHIN `window`. After that many crash restarts inside one
865/// window the module enters [`ModuleState::Failed`] and the supervisor stops
866/// the crash loop.
867///
868/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
869/// and that only survived because crashes were rare: a module that crashed
870/// three times across a week was disabled forever by crashes that had nothing
871/// to do with each other. That stopped being survivable once modules began
872/// exiting non-zero whenever the daemon's connection to them drops, because
873/// then every daemon-side connection drop spends a unit of the same budget and
874/// one flappy hour permanently stops a healthy module. Restarts older than
875/// `window` release their slot, so a module that crashed twice yesterday has a
876/// full budget today, while a genuine crash loop -- which is fast by
877/// definition -- still reaches the cap and stops.
878#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880    pub max_restarts: u32,
881    /// Base delay before a crash replacement. The actual delay escalates with
882    /// the number of recent crash replacements and is capped by `max_backoff`.
883    pub backoff: Duration,
884    /// Maximum delay before a crash replacement.
885    pub max_backoff: Duration,
886    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
887    /// budget effectively infinite (nothing is ever in-window), which is why
888    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
889    pub window: Duration,
890}
891
892impl RestartPolicy {
893    /// A policy with the default crash window. Callers that care about the
894    /// window say so with [`Self::with_window`]; the ones that do not are
895    /// asking for the standard rate limit, not for no limit.
896    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897        Self {
898            max_restarts,
899            backoff,
900            max_backoff: DEFAULT_MAX_BACKOFF,
901            window: DEFAULT_RESTART_WINDOW,
902        }
903    }
904
905    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906        self.max_backoff = max_backoff;
907        self
908    }
909
910    pub fn with_window(mut self, window: Duration) -> Self {
911        self.window = window;
912        self
913    }
914
915    /// Calculate the capped exponential delay for the next crash replacement.
916    /// `restart_in_window` is zero for the first replacement after an operator
917    /// action (restart, reload, re-enable) cleared the crash ring, or after all
918    /// older crash replacements have aged out of the window.
919    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920        if self.backoff.is_zero() || self.max_backoff.is_zero() {
921            return Duration::ZERO;
922        }
923
924        let mut delay = self.backoff;
925        for _ in 0..restart_in_window {
926            if delay >= self.max_backoff {
927                return self.max_backoff;
928            }
929            delay = delay
930                .checked_mul(10)
931                .unwrap_or(self.max_backoff)
932                .min(self.max_backoff);
933        }
934        delay.min(self.max_backoff)
935    }
936
937    /// The one sentence that explains a budget-exhausted stop, used for both the
938    /// log line and the terminal record so the two cannot drift. It names the
939    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
940    /// exactly what this budget is not.
941    fn budget_exhausted_detail(&self) -> String {
942        format!(
943            "crash budget exhausted: max_restarts={} within window_secs={}",
944            self.max_restarts,
945            self.window.as_secs()
946        )
947    }
948}
949
950impl Default for RestartPolicy {
951    fn default() -> Self {
952        Self {
953            max_restarts: DEFAULT_MAX_RESTARTS,
954            backoff: DEFAULT_BACKOFF,
955            max_backoff: DEFAULT_MAX_BACKOFF,
956            window: DEFAULT_RESTART_WINDOW,
957        }
958    }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963    restart_in_window: u32,
964    delay: Duration,
965}
966
967/// Whether the daemon itself will bring this module back after the exit being
968/// handled: it is enabled AND its in-window crash restarts are below the cap.
969///
970/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
971/// the window are dropped here rather than by a timer, so the count is right
972/// the moment somebody asks and no bookkeeping runs for idle modules.
973fn daemon_will_restart(
974    state: &mut SupervisorSnapshot,
975    policy: &RestartPolicy,
976    now: Instant,
977) -> bool {
978    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988    Report,
989    Restart,
990    Alert,
991}
992
993impl fmt::Display for HealthAction {
994    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995        f.write_str(match self {
996            Self::Report => "report",
997            Self::Restart => "restart",
998            Self::Alert => "alert",
999        })
1000    }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005    /// Optional loopback HTTP endpoint for a managed non-wire process.
1006    /// Changing it applies live on rescan; the process protocol changes only
1007    /// at its next spawn.
1008    pub http: Option<String>,
1009    pub cadence: Duration,
1010    pub deadline: Duration,
1011    pub failure_threshold: u32,
1012    pub on_degraded: HealthAction,
1013    pub on_failing: HealthAction,
1014    pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018    fn default() -> Self {
1019        Self {
1020            http: None,
1021            cadence: DEFAULT_HEALTH_CADENCE,
1022            deadline: DEFAULT_HEALTH_DEADLINE,
1023            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024            on_degraded: HealthAction::Report,
1025            on_failing: HealthAction::Report,
1026            critical: false,
1027        }
1028    }
1029}
1030
1031/// The supervisor's view of one module's health, relayed to clients over
1032/// channel-0 and rendered by `ck health`.
1033///
1034/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1035/// stated here rather than only at the wire type a consumer reads. A reader can
1036/// look up what `None` means; only a writer can silently change it, and the
1037/// writer has no reason to go looking at a downstream contract before editing.
1038///
1039/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1040/// back to `None` on re-registration precisely so a respawned module does not
1041/// carry its predecessor's timestamp — so an old value and an absent one call for
1042/// opposite readings, and anything that defaulted this to a number would make a
1043/// never-probed module indistinguishable from one probed at the epoch.
1044///
1045/// `detail` and `metrics` are `None` when the module published none on this
1046/// probe, which does not mean it reported nothing wrong — it is also the shape
1047/// when the probe never reached it. `last_probe_ms` is what separates those.
1048#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050    pub status: SupervisorHealthStatus,
1051    pub last_probe_ms: Option<u64>,
1052    pub detail: Option<String>,
1053    pub metrics: Option<Value>,
1054    pub consecutive_failures: u32,
1055    /// Number of replies received after a recurring health probe's deadline.
1056    /// Unlike a timeout, every increment proves the module was alive.
1057    pub late_answer_count: u64,
1058    /// End-to-end latency of the newest late reply, measured from probe start.
1059    pub last_late_answer_latency_ms: Option<u64>,
1060    pub last_action: Option<String>,
1061    /// Set together with `last_action`; the pair moves as one, and both being
1062    /// absent means no escalation has ever been taken rather than that the last
1063    /// one succeeded.
1064    pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068    fn default() -> Self {
1069        Self {
1070            status: SupervisorHealthStatus::Unknown,
1071            last_probe_ms: None,
1072            detail: None,
1073            metrics: None,
1074            consecutive_failures: 0,
1075            late_answer_count: 0,
1076            last_late_answer_latency_ms: None,
1077            last_action: None,
1078            last_action_ms: None,
1079        }
1080    }
1081}
1082
1083/// Typed lifecycle state for a supervised module.
1084#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086    Starting,
1087    Running,
1088    Unresponsive,
1089    Restarting,
1090    Draining,
1091    Stopped,
1092    Failed,
1093    Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098        f.write_str(match self {
1099            Self::Starting => "starting",
1100            Self::Running => "running",
1101            Self::Unresponsive => "unresponsive",
1102            Self::Restarting => "restarting",
1103            Self::Draining => "draining",
1104            Self::Stopped => "stopped",
1105            Self::Failed => "failed",
1106            Self::Disabled => "disabled",
1107        })
1108    }
1109}
1110
1111/// Supervisor classification of a child-process exit.
1112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114    Clean,
1115    Crash,
1116    DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120    fn from(kind: ExitKind) -> Self {
1121        match kind {
1122            ExitKind::Clean => Self::Clean,
1123            ExitKind::Crash => Self::Crash,
1124            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125        }
1126    }
1127}
1128
1129/// Exact process identity retained when a supervised module registers its
1130/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133    pub(crate) pid: u32,
1134    pub(crate) start_time: u64,
1135}
1136
1137/// Last observed child exit, if any.
1138#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140    pub kind: ExitKind,
1141    pub code: Option<i32>,
1142    pub signal: Option<i32>,
1143    pub at_ms: u64,
1144}
1145
1146/// Point-in-time module status answerable by subc without forwarding to the
1147/// module process.
1148#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150    pub module_id: String,
1151    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1152    pub waiting_for_previous_pid: Option<u32>,
1153    /// Why the previous instance still prevents a new start.
1154    pub waiting_for_previous_detail: Option<String>,
1155    /// The daemon's Windows per-spawn record: `pipe` after the process it
1156    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1157    /// List/status reports use this record, not the source the module sends
1158    /// in its registration message's manifest.provenance.launch_nonce_source.
1159    pub launch_nonce_source: Option<String>,
1160    pub state: ModuleState,
1161    pub enabled: bool,
1162    pub process_alive: bool,
1163    pub registration_active: bool,
1164    /// The module's declared wire protocol, carried beside `live` because it is
1165    /// what makes `live` readable: the two fields answer one question together.
1166    /// While a process is alive this is its launch declaration, not a later
1167    /// pending-reload edit. When down it is the configured next launch protocol.
1168    pub protocol: ModuleProtocol,
1169    /// Whether the module is serving, under the strongest definition the daemon
1170    /// can assert for its protocol.
1171    ///
1172    /// A subc module must also be REGISTERED: its process being alive says
1173    /// nothing about whether it can take a request. A `protocol: "none"` module
1174    /// never registers, so that term is dropped and this falls back to "enabled,
1175    /// running, and the process the daemon launched is alive" -- which is all
1176    /// the daemon observes about a process that speaks no subc wire. It stays a
1177    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1178    /// rather than printing it bare.
1179    pub live: bool,
1180    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1181    /// restarts have already released their slot, so this count can go down
1182    /// without anybody touching the module.
1183    pub restart_count: u32,
1184    /// Replacement processes spawned over this module's entire supervisor lifetime;
1185    /// unlike `restart_count`, this value is never reset by an operator action
1186    /// and never falls out of a window.
1187    pub lifetime_restarts: u32,
1188    pub spawn_generation: u64,
1189    /// The budget `restart_count` is spent against. Carried alongside the count
1190    /// because the count alone does not say how close the module is to being
1191    /// disabled, and reporting one without the other is what makes an
1192    /// about-to-be-retired module look ordinary.
1193    pub max_restarts: u32,
1194    /// The span `restart_count` is counted over. Carried with the pair above for
1195    /// the same reason they are carried together: "2 of 3" means one thing for a
1196    /// ten-minute window and something else entirely for a lifetime.
1197    pub restart_window: Duration,
1198    /// Effective drain and restart timing policy used by this running module.
1199    /// These values are carried together with the restart budget so status
1200    /// readers can compare configured intent with what the supervisor applied.
1201    pub drain_timeout: Duration,
1202    pub restart_backoff: Duration,
1203    pub restart_max_backoff: Duration,
1204    /// The module's process. On macOS this stays absent while the `ck-subc`
1205    /// launch trampoline is still running in that pid, and appears once the
1206    /// exec acknowledgement confirms the module image has replaced it. Launch
1207    /// time and the supervisor's own hold on the process are unaffected.
1208    pub pid: Option<u32>,
1209    pub spawned_at_ms: Option<u64>,
1210    pub spawned_from: Option<PathBuf>,
1211    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212    pub process_start_time: Option<u64>,
1213    pub last_exit: Option<ExitReport>,
1214    pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220    generation: u64,
1221    turn: u64,
1222    registered_connection: Option<ConnectionId>,
1223    next_probe_at: Option<Instant>,
1224    wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230    #[cfg(windows)]
1231    pipe_consumed: Option<Arc<OnceLock<()>>>,
1232    /// Counts running-child loop turns, not executor polls of a parked wait.
1233    #[cfg(test)]
1234    actor_turns: u64,
1235    /// Running select after exec confirmation and with no unconsumed registry
1236    /// event. Recording a spawn or starting a loop turn is not this barrier.
1237    #[cfg(test)]
1238    actor_select: Option<ActorSelectCheckpoint>,
1239    state: ModuleState,
1240    enabled: bool,
1241    process_alive: bool,
1242    spawned_protocol: Option<ModuleProtocol>,
1243    spawn_failure: Option<String>,
1244    waiting_for_previous_pid: Option<u32>,
1245    waiting_for_previous_detail: Option<String>,
1246    /// When each crash restart was spent, oldest first. This IS the crash
1247    /// budget: its in-window length is the count an operator sees and the count
1248    /// the restart decision is made against, so there is no second counter that
1249    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1250    /// operator actions that used to zero the old lifetime counter.
1251    crash_restarts: VecDeque<Instant>,
1252    lifetime_restarts: u32,
1253    /// Successful child spawns in this daemon incarnation.
1254    ///
1255    /// `lifetime_restarts` was considered and rejected: it starts at zero
1256    /// (line 640), successful initial/operator spawns in `set_running` do not
1257    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1258    /// increments before a successful replacement exists (lines 604, 3846,
1259    /// and 3921), so a failed spawn can consume it. This counter moves only
1260    /// when a live PID is accepted below.
1261    spawn_generation: u64,
1262    pid: Option<u32>,
1263    #[cfg(target_os = "macos")]
1264    report_ready: Option<Arc<OnceLock<()>>>,
1265    /// Last reaped child, retained after current process facts are cleared.
1266    reaped_pid: Option<u32>,
1267    /// Whether a replacement has already been scheduled for the current process.
1268    respawn_pending: bool,
1269    /// A second restart is waiting for the replacement already scheduled.
1270    coalesced_restart_pending: bool,
1271    spawned_at_ms: Option<u64>,
1272    spawned_from: Option<PathBuf>,
1273    launch_spec_digest: Option<LaunchSpecDigest>,
1274    spawned_file_identity: Option<SpawnedFileIdentity>,
1275    process_start_time: Option<u64>,
1276    deliberate_severance: Option<ProcessIdentity>,
1277    last_exit: Option<ExitReport>,
1278    /// Diagnostic attached to the next drain's terminal record, if any.
1279    drain_disposition_detail: Option<String>,
1280    health: ModuleHealthStatus,
1281    /// Whether the current process was started as a swap candidate and so
1282    /// lives in the module's alternate cgroup. The next swap's candidate takes
1283    /// the other one, so the two processes of a swap never share a cgroup. A
1284    /// plain spawn always uses the primary cgroup.
1285    in_alternate_slot: bool,
1286    /// Whether the current `Draining` state ends in a replacement process
1287    /// (restart, reload, health restart) rather than a stop. Only meaningful
1288    /// while `state` is `Draining`; every entry into that state rewrites it.
1289    /// It is what lets route.open answer the retryable `module_reloading` to a
1290    /// consumer that reaches a still-registered process mid-restart, instead of
1291    /// the `supervisor_not_live` a stop or disable deserves.
1292    draining_to_replace: bool,
1293    /// Whether a configuration update has been applied since the current
1294    /// process was spawned, so that process runs an older spec than the one
1295    /// the supervisor now holds. A queued restart is only coalesced into a
1296    /// fresher process when this is false: a restart requested to pick up a
1297    /// new configuration must not be satisfied by a process that predates it.
1298    configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302    /// The pid that status, provenance and resource readings may report. While
1303    /// the launch trampoline still runs in the pid, reading its executable or
1304    /// resource use would describe `ck-subc`, not the module, so none is
1305    /// reported until the exec acknowledgement confirms the module image.
1306    fn reported_pid(&self) -> Option<u32> {
1307        #[cfg(target_os = "macos")]
1308        if self
1309            .report_ready
1310            .as_ref()
1311            .is_some_and(|ready| ready.get().is_none())
1312        {
1313            return None;
1314        }
1315        self.pid
1316    }
1317
1318    fn starting() -> Self {
1319        Self::new(ModuleState::Starting, true)
1320    }
1321
1322    fn disabled() -> Self {
1323        Self::new(ModuleState::Disabled, false)
1324    }
1325
1326    /// Crash restarts still inside `window`, having dropped the ones that are
1327    /// not. Pruning on read is what makes the budget a rate: an instant older
1328    /// than the window stops holding a slot the moment anybody counts.
1329    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330        while let Some(oldest) = self.crash_restarts.front() {
1331            if now.duration_since(*oldest) > window {
1332                self.crash_restarts.pop_front();
1333            } else {
1334                break;
1335            }
1336        }
1337        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338    }
1339
1340    /// Spend one unit of the crash budget and record the restart in the ledger.
1341    ///
1342    /// The ring is bounded by the cap because more than `max_restarts` in-window
1343    /// instants can never be reached (the caller refuses the restart first), so
1344    /// anything beyond that is an unbounded queue waiting to happen.
1345    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346        self.crash_restarts.push_back(now);
1347        while self.crash_restarts.len() > policy.max_restarts as usize {
1348            self.crash_restarts.pop_front();
1349        }
1350        self.lifetime_restarts += 1;
1351    }
1352
1353    /// Reserve one crash-restart slot and calculate the delay before respawning.
1354    /// The count is captured before recording this restart, so the first retry
1355    /// uses the base delay and each later in-window retry escalates once.
1356    fn next_crash_restart(
1357        &mut self,
1358        policy: &RestartPolicy,
1359        now: Instant,
1360    ) -> Option<CrashRestartSchedule> {
1361        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362        if restart_in_window >= policy.max_restarts {
1363            return None;
1364        }
1365        self.record_crash_restart(policy, now);
1366        Some(CrashRestartSchedule {
1367            restart_in_window,
1368            delay: policy.delay_for_restart(restart_in_window),
1369        })
1370    }
1371
1372    /// Give the module its full budget back, as an operator restart, reload, or
1373    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1374    /// ledger of what actually happened, and an operator action does not unmake
1375    /// the crashes.
1376    fn clear_crash_restarts(&mut self) {
1377        self.crash_restarts.clear();
1378    }
1379
1380    fn new(state: ModuleState, enabled: bool) -> Self {
1381        Self {
1382            exec_outcome: None,
1383            #[cfg(test)]
1384            actor_turns: 0,
1385            #[cfg(test)]
1386            actor_select: None,
1387            state,
1388            enabled,
1389            process_alive: false,
1390            #[cfg(windows)]
1391            pipe_consumed: None,
1392            spawned_protocol: None,
1393            spawn_failure: None,
1394            waiting_for_previous_pid: None,
1395            waiting_for_previous_detail: None,
1396            crash_restarts: VecDeque::new(),
1397            lifetime_restarts: 0,
1398            spawn_generation: 0,
1399            pid: None,
1400            #[cfg(target_os = "macos")]
1401            report_ready: None,
1402            reaped_pid: None,
1403            respawn_pending: false,
1404            coalesced_restart_pending: false,
1405            spawned_at_ms: None,
1406            spawned_from: None,
1407            launch_spec_digest: None,
1408            spawned_file_identity: None,
1409            process_start_time: None,
1410            deliberate_severance: None,
1411            last_exit: None,
1412            drain_disposition_detail: None,
1413            health: ModuleHealthStatus::default(),
1414            in_alternate_slot: false,
1415            draining_to_replace: false,
1416            configuration_updated_since_spawn: false,
1417        }
1418    }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427    version: u8,
1428    frames: mpsc::Sender<Frame>,
1429    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1430    /// from which event. The full frame channel cannot carry that news, so it
1431    /// travels beside it; see `SpawnEventFeed::subscribe`.
1432    lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437    daemon_incarnation: String,
1438    seq: u64,
1439    capacity: usize,
1440    live: HashMap<String, LiveSpawn>,
1441    generations: HashMap<String, u64>,
1442    events: VecDeque<SpawnEvent>,
1443    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447    fn default() -> Self {
1448        Self {
1449            daemon_incarnation: "unconfigured".to_string(),
1450            seq: 0,
1451            capacity: SPAWN_EVENT_RING_CAPACITY,
1452            live: HashMap::new(),
1453            generations: HashMap::new(),
1454            events: VecDeque::new(),
1455            subscribers: HashMap::new(),
1456        }
1457    }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462    Arc<Mutex<SpawnEventState>>,
1463    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468    ForeignIncarnation { current: String },
1469    TooOld { oldest: SpawnCursor },
1470    Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474    fn configure_incarnation(&self, daemon_incarnation: String) {
1475        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476        state.daemon_incarnation = daemon_incarnation;
1477        state.seq = 0;
1478        state.live.clear();
1479        state.generations.clear();
1480        state.events.clear();
1481        state.subscribers.clear();
1482    }
1483
1484    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485        SpawnCursor {
1486            daemon_incarnation: state.daemon_incarnation.clone(),
1487            seq: state.seq,
1488        }
1489    }
1490
1491    fn snapshot(&self) -> SpawnSnapshot {
1492        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495        SpawnSnapshot {
1496            cursor: Self::cursor(&state),
1497            ring_bound: state.capacity as u64,
1498            live,
1499        }
1500    }
1501
1502    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504        let generation = state
1505            .generations
1506            .get(module_id)
1507            .copied()
1508            .unwrap_or(0)
1509            .checked_add(1)
1510            .expect("spawn generation exhausted");
1511        state.generations.insert(module_id.to_string(), generation);
1512        let live = LiveSpawn {
1513            module_id: module_id.to_string(),
1514            spawn_generation: generation,
1515            pid,
1516            spawned_at_ms,
1517        };
1518        state.live.insert(module_id.to_string(), live);
1519        Self::emit_locked(
1520            &mut state,
1521            SpawnEventKind::Spawned,
1522            module_id.to_string(),
1523            generation,
1524            pid,
1525            None,
1526            None,
1527        );
1528        generation
1529    }
1530
1531    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533        let Some(live) = state.live.remove(module_id) else {
1534            warn!(
1535                module_id,
1536                "terminal record had no live spawn event identity"
1537            );
1538            return;
1539        };
1540        Self::emit_locked(
1541            &mut state,
1542            SpawnEventKind::Exited,
1543            module_id.to_string(),
1544            live.spawn_generation,
1545            live.pid,
1546            exit_code,
1547            exit_signal,
1548        );
1549    }
1550
1551    /// Report the exit of a process that a swap has already replaced.
1552    ///
1553    /// `emit_exited` removes the module's live entry, which after a swap's
1554    /// cutover describes the promoted candidate, not the old process now
1555    /// exiting. This emits the old generation's exit and leaves the live entry
1556    /// alone unless it still names that generation.
1557    fn emit_superseded_exited(
1558        &self,
1559        module_id: &str,
1560        spawn_generation: u64,
1561        pid: u32,
1562        exit_code: Option<i32>,
1563        exit_signal: Option<i32>,
1564    ) {
1565        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566        if state
1567            .live
1568            .get(module_id)
1569            .is_some_and(|live| live.spawn_generation == spawn_generation)
1570        {
1571            state.live.remove(module_id);
1572        }
1573        Self::emit_locked(
1574            &mut state,
1575            SpawnEventKind::Exited,
1576            module_id.to_string(),
1577            spawn_generation,
1578            pid,
1579            exit_code,
1580            exit_signal,
1581        );
1582    }
1583
1584    #[allow(clippy::too_many_arguments)]
1585    fn emit_locked(
1586        state: &mut SpawnEventState,
1587        kind: SpawnEventKind,
1588        module_id: String,
1589        spawn_generation: u64,
1590        pid: u32,
1591        exit_code: Option<i32>,
1592        exit_signal: Option<i32>,
1593    ) {
1594        state.seq = state
1595            .seq
1596            .checked_add(1)
1597            .expect("spawn event sequence exhausted");
1598        let event = SpawnEvent {
1599            cursor: Self::cursor(state),
1600            kind,
1601            module_id,
1602            spawn_generation,
1603            pid,
1604            exit_code,
1605            exit_signal,
1606        };
1607        state.events.push_back(event.clone());
1608        while state.events.len() > state.capacity {
1609            state.events.pop_front();
1610        }
1611        let body = match serde_json::to_vec(&event) {
1612            Ok(body) => body,
1613            Err(error) => {
1614                error!(%error, "failed to serialize supervisor spawn event");
1615                return;
1616            }
1617        };
1618        state.subscribers.retain(|(connection_id, corr), subscriber| {
1619            let frame = Frame::build_with_version(
1620                subscriber.version,
1621                FrameType::StreamData,
1622                control_flags(),
1623                0,
1624                0,
1625                *corr,
1626                body.clone(),
1627            );
1628            match frame {
1629                Ok(frame) => {
1630                    if subscriber.frames.try_send(frame).is_ok() {
1631                        true
1632                    } else {
1633                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634                        if let Some(lagged) = subscriber.lagged.take() {
1635                            let _ = lagged.send(event.cursor.clone());
1636                        }
1637                        false
1638                    }
1639                }
1640                Err(error) => {
1641                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642                    false
1643                }
1644            }
1645        });
1646    }
1647
1648    fn subscribe(
1649        &self,
1650        connection_id: ConnectionId,
1651        corr: u64,
1652        version: u8,
1653        since: Option<SpawnCursor>,
1654        sink: FrameSink,
1655    ) -> Result<(), SpawnSubscribeRefusal> {
1656        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658        {
1659            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660            let replay = if let Some(since) = since {
1661                if since.daemon_incarnation != state.daemon_incarnation {
1662                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663                        current: state.daemon_incarnation.clone(),
1664                    });
1665                }
1666                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667                    if since.seq < oldest.seq.saturating_sub(1) {
1668                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669                    }
1670                }
1671                state
1672                    .events
1673                    .iter()
1674                    .filter(|event| event.cursor.seq > since.seq)
1675                    .cloned()
1676                    .collect::<Vec<_>>()
1677            } else {
1678                Vec::new()
1679            };
1680            for event in replay {
1681                let body = serde_json::to_vec(&event)
1682                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683                let frame = Frame::build_with_version(
1684                    version,
1685                    FrameType::StreamData,
1686                    control_flags(),
1687                    0,
1688                    0,
1689                    corr,
1690                    body,
1691                )
1692                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693                frames
1694                    .try_send(frame)
1695                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696            }
1697            state.subscribers.insert(
1698                (connection_id, corr),
1699                SpawnSubscriber {
1700                    version,
1701                    frames,
1702                    lagged: Some(lagged),
1703                },
1704            );
1705        }
1706        // The lagged terminal is sent here, by the forwarder, rather than by
1707        // the emitter: at the moment of the drop the subscriber's own channel
1708        // is full, and writing to the connection sink directly from the emitter
1709        // would put the Error AHEAD of the events still queued in that channel
1710        // (and the emitter holds the feed lock, so it cannot await the sink).
1711        // Dropping the subscriber drops the only sender, so `recv` drains every
1712        // queued event and then returns `None`; only then is the Error sent, so
1713        // the client sees each event it can keep, then the reason it was cut.
1714        // Cancel and connection removal drop the oneshot unsent, so they end
1715        // the stream with no Error.
1716        tokio::spawn(async move {
1717            while let Some(frame) = receiver.recv().await {
1718                if sink.send(frame).await.is_err() {
1719                    return;
1720                }
1721            }
1722            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723                return;
1724            };
1725            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726                Ok(frame) => {
1727                    let _ = sink.send(frame).await;
1728                }
1729                Err(error) => {
1730                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731                }
1732            }
1733        });
1734        Ok(())
1735    }
1736
1737    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738        let Some(subscriber) = self
1739            .0
1740            .lock()
1741            .unwrap_or_else(|p| p.into_inner())
1742            .subscribers
1743            .remove(&(connection_id, corr))
1744        else {
1745            return false;
1746        };
1747        if let Ok(frame) = Frame::build_with_version(
1748            subscriber.version,
1749            FrameType::StreamEnd,
1750            control_flags(),
1751            0,
1752            0,
1753            corr,
1754            Vec::new(),
1755        ) {
1756            tokio::spawn(async move {
1757                let _ = subscriber.frames.send(frame).await;
1758            });
1759        }
1760        true
1761    }
1762
1763    fn remove_connection(&self, connection_id: ConnectionId) {
1764        self.0
1765            .lock()
1766            .unwrap_or_else(|p| p.into_inner())
1767            .subscribers
1768            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769    }
1770
1771    #[cfg(any(test, feature = "test-support"))]
1772    fn set_capacity(&self, capacity: usize) {
1773        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774    }
1775
1776    #[cfg(any(test, feature = "test-support"))]
1777    fn subscriber_count(&self) -> usize {
1778        self.0
1779            .lock()
1780            .unwrap_or_else(|p| p.into_inner())
1781            .subscribers
1782            .len()
1783    }
1784}
1785
1786/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1787/// The terminal Error a lagged spawn subscriber receives after its queued events.
1788fn spawn_subscriber_lagged_frame(
1789    version: u8,
1790    corr: u64,
1791    first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796            .to_string(),
1797        detail: Some(serde_json::json!({
1798            "first_undelivered_cursor": first_undelivered
1799        })),
1800    })
1801    .map_err(|error| error.to_string())?;
1802    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803        .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807    fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809    /// Whether the supervisor is replacing this module's process right now: an
1810    /// operator restart or reload, a health restart, or a crash respawn whose
1811    /// backoff is running. A module in that state is not live, but a consumer
1812    /// refused now should retry shortly rather than treat the target as gone.
1813    /// Stopped, failed, and disabled modules are not replacing.
1814    fn process_replacing(&self, _module_id: &str) -> bool {
1815        false
1816    }
1817}
1818
1819/// Shared process-liveness registry keyed by supervised `module_id`.
1820#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826    pub fn new() -> Self {
1827        Self::default()
1828    }
1829
1830    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831        let mut snapshots = self
1832            .snapshots
1833            .lock()
1834            .unwrap_or_else(|poisoned| poisoned.into_inner());
1835        snapshots.insert(module_id, snapshot);
1836    }
1837
1838    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        let is_current = snapshots
1844            .get(module_id)
1845            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846            .unwrap_or(false);
1847        if is_current {
1848            snapshots.remove(module_id);
1849        }
1850    }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854    fn process_live(&self, module_id: &str) -> Option<bool> {
1855        let snapshot = {
1856            let snapshots = self
1857                .snapshots
1858                .lock()
1859                .unwrap_or_else(|poisoned| poisoned.into_inner());
1860            snapshots.get(module_id).cloned()
1861        }?;
1862        let snapshot = snapshot
1863            .lock()
1864            .unwrap_or_else(|poisoned| poisoned.into_inner());
1865        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866    }
1867
1868    fn process_replacing(&self, module_id: &str) -> bool {
1869        let Some(snapshot) = self
1870            .snapshots
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner())
1873            .get(module_id)
1874            .cloned()
1875        else {
1876            return false;
1877        };
1878        let snapshot = snapshot
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner());
1881        snapshot.enabled
1882            && match snapshot.state {
1883                ModuleState::Restarting => true,
1884                ModuleState::Draining => snapshot.draining_to_replace,
1885                ModuleState::Starting
1886                | ModuleState::Running
1887                | ModuleState::Unresponsive
1888                | ModuleState::Stopped
1889                | ModuleState::Failed
1890                | ModuleState::Disabled => false,
1891            }
1892    }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898    reached: tokio::sync::Notify,
1899    resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904    Spawn,
1905    Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910    deadline: Instant,
1911    kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918    start_requester: Option<Principal>,
1919    pending_activity_restart: bool,
1920    provenance_probe: ExecutableIdentityProbe,
1921    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1922    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923    /// A reload acknowledges completion only after its replacement registers.
1924    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925    restart_policy: RestartPolicy,
1926    /// This module's RESOLVED drain budget: per-module config when present,
1927    /// else `default_drain_timeout`.
1928    drain_timeout: Duration,
1929    /// Shared with the status handle so the attested value changes atomically
1930    /// when a rescan updates the running drain policy.
1931    effective_drain_timeout: Arc<Mutex<Duration>>,
1932    /// The supervisor-wide fallback, kept so a configuration update that
1933    /// REMOVES the per-module override can re-resolve to it.
1934    default_drain_timeout: Duration,
1935    health: HealthConfig,
1936    connection_file_path: Option<PathBuf>,
1937    capture_logs_dir: Option<PathBuf>,
1938    forwarding: Option<Arc<ForwardingTable>>,
1939    /// The shared handle, so every spawn path (initial, restart, reload) records the
1940    /// reserved-module launch nonce the HELLO verifier checks against.
1941    supervisor_handle: Option<SupervisorHandle>,
1942    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1943    /// status queries.
1944    ///
1945    /// One ring per module, held across every respawn. The lines explaining an exit
1946    /// are written BEFORE that exit, so a ring recreated per process would be empty
1947    /// exactly when it is asked for.
1948    stderr_ring: Arc<Mutex<StderrRing>>,
1949    terminal_ring: Arc<Mutex<TerminalRing>>,
1950    spawn_events: SpawnEventFeed,
1951    child_roster: ChildRoster,
1952    #[cfg(target_os = "linux")]
1953    cgroup_placement: Option<subc_cgroup::Placement>,
1954    #[cfg(test)]
1955    test_seed_stale_facts_before_enable_spawn: bool,
1956    #[cfg(test)]
1957    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962    spec: ModuleSpec,
1963    health: HealthConfig,
1964}
1965
1966/// Shared daemon lookup table for supervised module handles.
1967///
1968/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1969/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1970/// launch nonces recorded at spawn are checked by the same daemon instance.
1971#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976    /// Module ids the supervisor has taken on. An id is added BEFORE the
1977    /// module's first process is spawned and removed only when the module
1978    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1979    /// the keys of `modules`.
1980    ///
1981    /// `modules` cannot answer "is this module configured?" on its own: a
1982    /// [`SupervisedModule`] only exists once its process has been spawned, and
1983    /// a fast child can connect, register, sync its scopes and ask about them
1984    /// before the supervisor has inserted it. Answering "not configured" in that
1985    /// gap makes scope admission refuse with the terminal "will never sync"
1986    /// instead of the retryable "has not synced yet".
1987    configured_ids: Arc<Mutex<HashSet<String>>>,
1988    spawn_events: SpawnEventFeed,
1989    /// The current expected launch nonce for each reserved module_id. Set when the
1990    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1991    /// non-reserved module never has an entry here and is never nonce-checked.
1992    /// Reserved module ids and the nonce that authorizes their next HELLO.
1993    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1994    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1995    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1996    /// had NO entry and admitted anyone: the reservation protected the nonce
1997    /// holder, not the NAME (found live by CKCRED's canary probe registering
1998    /// against a reserved scratch id).
1999    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2001    ///
2002    /// This is deliberately in-memory only: subc is state-free across daemon
2003    /// restarts, and the tombstone only explains the hours-after-removal window
2004    /// while this executing daemon is still alive. Do not persist it in a store.
2005    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006    /// The current launch nonce for every supervised spawn. This is separate from
2007    /// reserved_nonces because consumer route.open attestation applies to all spawned
2008    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2009    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010    /// Reserved namespace prefixes mapped to the supervised owner module whose
2011    /// current spawn nonce authorizes HELLO claims below the prefix.
2012    ///
2013    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2014    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2015    /// accidental collisions and lower-trust processes from squatting protected
2016    /// namespaces.
2017    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018    /// Blue/green swaps in progress, by module id. An entry exists from just
2019    /// before the candidate process is spawned until the swap has failed, or
2020    /// has cut over and the old process is gone. While it exists, HELLO for the
2021    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2022    /// consumer attestation accepts both processes' nonces.
2023    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2025    promotion_observer: PromotionObserverSlot,
2026    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2027    /// this daemon-wide ordering, a rescan could retire or update a module while a
2028    /// concurrent reload still held its old handle and launch specification.
2029    operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032/// Told when a swap has promoted its candidate to be the module's active
2033/// registration.
2034///
2035/// An ordinary HELLO runs the control plane's registration side effects (the
2036/// capability cache, the deny census, the requirement recompute) as it
2037/// registers. A swap candidate's HELLO does not, because it is not routable;
2038/// promotion is when those must run instead, and promotion happens in the
2039/// supervisor, which has no other way into the control handler.
2040pub(crate) trait SwapPromotionObserver: Send + Sync {
2041    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2045/// control handler) owns this handle, so a strong reference back would be a
2046/// cycle that keeps both alive.
2047#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052        f.write_str("PromotionObserverSlot")
2053    }
2054}
2055
2056/// The nonces of one open swap.
2057#[derive(Debug, Clone)]
2058struct OpenSwap {
2059    /// The launch nonce minted for the candidate process. It is the swap
2060    /// token: the only thing that admits a HELLO into the candidate slot.
2061    candidate_nonce: String,
2062    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2063    /// here because cutover moves the module's recorded spawn nonce to the
2064    /// candidate while the incumbent is still draining and its consumers are
2065    /// still attesting with this one.
2066    incumbent_nonce: Option<String>,
2067    /// Set once a HELLO has been admitted with the swap token, so the token
2068    /// admits one registration and cannot be replayed after cutover empties
2069    /// the candidate slot.
2070    candidate_admitted: bool,
2071}
2072
2073/// What the swap gate says about a HELLO. See
2074/// [`SupervisorHandle::swap_hello_admission`].
2075#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077    /// No swap is open for the id (or the HELLO carries the incumbent's own
2078    /// nonce); the ordinary gates decide.
2079    NotSwapping,
2080    /// The HELLO carries the swap token: register it into the candidate slot.
2081    Candidate,
2082    /// A swap is open and the HELLO carries a nonce the supervisor did not
2083    /// mint for this id, no nonce, or a token already used.
2084    Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089    Exact {
2090        module_id: String,
2091    },
2092    Prefix {
2093        prefix: String,
2094        owner_module_id: String,
2095    },
2096}
2097
2098impl SupervisorHandle {
2099    pub fn new() -> Self {
2100        Self::default()
2101    }
2102
2103    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104        self.spawn_events.snapshot()
2105    }
2106
2107    pub(crate) fn subscribe_spawns(
2108        &self,
2109        connection_id: ConnectionId,
2110        corr: u64,
2111        version: u8,
2112        since: Option<SpawnCursor>,
2113        sink: FrameSink,
2114    ) -> Result<(), SpawnSubscribeRefusal> {
2115        self.spawn_events
2116            .subscribe(connection_id, corr, version, since, sink)
2117    }
2118
2119    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120        self.spawn_events.cancel(connection_id, corr)
2121    }
2122
2123    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124        self.spawn_events.remove_connection(connection_id);
2125    }
2126
2127    #[cfg(any(test, feature = "test-support"))]
2128    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129        assert!(capacity > 0, "spawn event capacity must be non-zero");
2130        self.spawn_events.set_capacity(capacity);
2131    }
2132
2133    #[cfg(any(test, feature = "test-support"))]
2134    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135        self.spawn_events.subscriber_count()
2136    }
2137
2138    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2139    /// a respawn invalidates stale consumer identities.
2140    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141        self.spawn_nonces
2142            .lock()
2143            .unwrap_or_else(|poisoned| poisoned.into_inner())
2144            .insert(module_id.to_string(), nonce);
2145    }
2146
2147    /// Record the launch nonce expected from the next HELLO for a reserved module,
2148    /// replacing any prior nonce (a respawn invalidates the previous one).
2149    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150        self.reserved_nonces
2151            .lock()
2152            .unwrap_or_else(|poisoned| poisoned.into_inner())
2153            .insert(module_id.to_string(), Some(nonce));
2154    }
2155
2156    /// Record namespace prefixes owned by a supervised module.
2157    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158        let mut owners = self
2159            .reserved_prefix_owners
2160            .lock()
2161            .unwrap_or_else(|poisoned| poisoned.into_inner());
2162        owners.retain(|_, owner| owner != owner_module_id);
2163        for prefix in prefixes {
2164            owners.insert(prefix.clone(), owner_module_id.to_string());
2165        }
2166    }
2167
2168    /// The launch nonce most recently minted for a module's spawn, if any.
2169    #[cfg(test)]
2170    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171        self.spawn_nonces
2172            .lock()
2173            .unwrap_or_else(|poisoned| poisoned.into_inner())
2174            .get(module_id)
2175            .cloned()
2176    }
2177
2178    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180        let spawn_nonce = self
2181            .spawn_nonces
2182            .lock()
2183            .unwrap_or_else(|poisoned| poisoned.into_inner())
2184            .get(&spec.module_id)
2185            .cloned();
2186        let mut reserved_nonces = self
2187            .reserved_nonces
2188            .lock()
2189            .unwrap_or_else(|poisoned| poisoned.into_inner());
2190        if spec.reserved {
2191            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2192            // reserved name whose module has never spawned has no legitimate
2193            // holder, and the entry's absence is what used to leave the name
2194            // open to the first claimant.
2195            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196        }
2197        drop(reserved_nonces);
2198        // A later unreserved declaration must not silently unreserve an id that
2199        // was retained after its reserved configuration was removed. The explicit
2200        // release ceremony is the only operation that retires that gate.
2201        self.removal_tombstones
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .remove(&spec.module_id);
2205    }
2206
2207    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2208    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2209    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2210    /// with no matching prefix are always authorized.
2211    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212        self.reserved_hello_rejection(module_id, presented)
2213            .is_none()
2214    }
2215
2216    pub(crate) fn reserved_hello_rejection(
2217        &self,
2218        module_id: &str,
2219        presented: Option<&str>,
2220    ) -> Option<ReservedHelloRejection> {
2221        let nonces = self
2222            .reserved_nonces
2223            .lock()
2224            .unwrap_or_else(|poisoned| poisoned.into_inner());
2225        if let Some(expected) = nonces.get(module_id) {
2226            // `None` = reserved with no legitimate holder: refuse every
2227            // presentation, because no process can hold a nonce that was never
2228            // minted. Only a real minted nonce admits, in constant time.
2229            let authorized = match expected {
2230                Some(expected) => {
2231                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232                }
2233                None => false,
2234            };
2235            if authorized {
2236                return None;
2237            }
2238            return Some(ReservedHelloRejection::Exact {
2239                module_id: module_id.to_string(),
2240            });
2241        }
2242        drop(nonces);
2243
2244        let matched_prefix = self
2245            .reserved_prefix_owners
2246            .lock()
2247            .unwrap_or_else(|poisoned| poisoned.into_inner())
2248            .iter()
2249            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250            .max_by_key(|(prefix, _)| prefix.len())
2251            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252        let (prefix, owner_module_id) = matched_prefix?;
2253
2254        let authorized = presented.is_some_and(|presented| {
2255            self.spawn_nonces
2256                .lock()
2257                .unwrap_or_else(|poisoned| poisoned.into_inner())
2258                .get(&owner_module_id)
2259                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260                // While the owner is being swapped, children started by
2261                // either of its two processes hold that process's nonce.
2262                || self.swap_nonce_matches(&owner_module_id, presented)
2263        });
2264        if authorized {
2265            None
2266        } else {
2267            Some(ReservedHelloRejection::Prefix {
2268                prefix,
2269                owner_module_id,
2270            })
2271        }
2272    }
2273
2274    /// Whether a consumer connection proved it came from a daemon-spawned module.
2275    ///
2276    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2277    /// accepted only for module ids the supervisor has spawned.
2278    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279        if presented.is_empty() {
2280            return false;
2281        }
2282        let nonces = self
2283            .spawn_nonces
2284            .lock()
2285            .unwrap_or_else(|poisoned| poisoned.into_inner());
2286        let current = nonces
2287            .get(module_id)
2288            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289        drop(nonces);
2290        // During a swap two processes of the module are alive, and a consumer
2291        // started by either one presents that process's nonce. Accepting only
2292        // the recorded one would fail the incumbent's consumers for the whole
2293        // overlap once cutover moves the record to the candidate.
2294        current || self.swap_nonce_matches(module_id, presented)
2295    }
2296
2297    /// Whether `presented` is either nonce of an open swap for `module_id`.
2298    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299        let swaps = self
2300            .swaps
2301            .lock()
2302            .unwrap_or_else(|poisoned| poisoned.into_inner());
2303        swaps.get(module_id).is_some_and(|swap| {
2304            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307                })
2308        })
2309    }
2310
2311    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2312    /// Called before the candidate process exists.
2313    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314        let incumbent_nonce = self
2315            .spawn_nonces
2316            .lock()
2317            .unwrap_or_else(|poisoned| poisoned.into_inner())
2318            .get(module_id)
2319            .cloned();
2320        self.swaps
2321            .lock()
2322            .unwrap_or_else(|poisoned| poisoned.into_inner())
2323            .insert(
2324                module_id.to_string(),
2325                OpenSwap {
2326                    candidate_nonce,
2327                    incumbent_nonce,
2328                    candidate_admitted: false,
2329                },
2330            );
2331    }
2332
2333    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2334    /// the module's recorded one.
2335    pub(crate) fn close_swap(&self, module_id: &str) {
2336        self.swaps
2337            .lock()
2338            .unwrap_or_else(|poisoned| poisoned.into_inner())
2339            .remove(module_id);
2340    }
2341
2342    /// Install the observer told about swap promotions, replacing any earlier
2343    /// one.
2344    pub(crate) fn set_swap_promotion_observer(
2345        &self,
2346        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347    ) {
2348        *self
2349            .promotion_observer
2350            .0
2351            .lock()
2352            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353    }
2354
2355    /// Tell the installed observer, if it is still alive, that a swap promoted
2356    /// `registration`.
2357    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358        let observer = self
2359            .promotion_observer
2360            .0
2361            .lock()
2362            .unwrap_or_else(|poisoned| poisoned.into_inner())
2363            .as_ref()
2364            .and_then(std::sync::Weak::upgrade);
2365        if let Some(observer) = observer {
2366            observer.swap_promoted(registration);
2367        }
2368    }
2369
2370    /// Whether a swap is open for `module_id`.
2371    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372        self.swaps
2373            .lock()
2374            .unwrap_or_else(|poisoned| poisoned.into_inner())
2375            .contains_key(module_id)
2376    }
2377
2378    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2379    /// respawn would, once cutover has made the candidate the module's process.
2380    /// The swap stays open so the incumbent's nonce keeps attesting until the
2381    /// incumbent has drained and exited.
2382    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383        let candidate_nonce = self
2384            .swaps
2385            .lock()
2386            .unwrap_or_else(|poisoned| poisoned.into_inner())
2387            .get(module_id)
2388            .map(|swap| swap.candidate_nonce.clone());
2389        let Some(nonce) = candidate_nonce else {
2390            return;
2391        };
2392        self.set_spawn_nonce(module_id, nonce.clone());
2393        if reserved {
2394            self.set_reserved_nonce(module_id, nonce);
2395        }
2396    }
2397
2398    /// The swap gate for a HELLO claiming `module_id`.
2399    ///
2400    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2401    /// presents the candidate nonce, which the reserved gate (holding the
2402    /// incumbent's nonce) would refuse as `reserved_module` before swap
2403    /// admission was ever reached. And it applies to unreserved ids too: for an
2404    /// unreserved id the only thing that ever stopped a second process claiming
2405    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2406    /// refusal a swap lifts for its candidate.
2407    ///
2408    /// The incumbent's own nonce falls through to the ordinary gates, which
2409    /// treat it as they always have (a live incumbent is refused as a
2410    /// duplicate). Anything else while a swap is open is refused, including an
2411    /// absent nonce.
2412    pub(crate) fn swap_hello_admission(
2413        &self,
2414        module_id: &str,
2415        presented: Option<&str>,
2416    ) -> SwapHelloAdmission {
2417        let swaps = self
2418            .swaps
2419            .lock()
2420            .unwrap_or_else(|poisoned| poisoned.into_inner());
2421        let Some(swap) = swaps.get(module_id) else {
2422            return SwapHelloAdmission::NotSwapping;
2423        };
2424        let Some(presented) = presented else {
2425            return SwapHelloAdmission::Refused;
2426        };
2427        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428            return if swap.candidate_admitted {
2429                SwapHelloAdmission::Refused
2430            } else {
2431                SwapHelloAdmission::Candidate
2432            };
2433        }
2434        if swap
2435            .incumbent_nonce
2436            .as_deref()
2437            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438        {
2439            return SwapHelloAdmission::NotSwapping;
2440        }
2441        SwapHelloAdmission::Refused
2442    }
2443
2444    /// Record that the swap token has registered a candidate, so it admits no
2445    /// second HELLO.
2446    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447        if let Some(swap) = self
2448            .swaps
2449            .lock()
2450            .unwrap_or_else(|poisoned| poisoned.into_inner())
2451            .get_mut(module_id)
2452        {
2453            swap.candidate_admitted = true;
2454        }
2455    }
2456
2457    /// Test/support lookup for the current launch nonce of a supervised spawn.
2458    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459        self.spawn_nonces
2460            .lock()
2461            .unwrap_or_else(|poisoned| poisoned.into_inner())
2462            .get(module_id)
2463            .cloned()
2464    }
2465
2466    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2467    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468        self.reserved_nonces
2469            .lock()
2470            .unwrap_or_else(|poisoned| poisoned.into_inner())
2471            .get(module_id)
2472            .cloned()
2473            .flatten()
2474    }
2475
2476    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477        // Normally already marked before the process was spawned; marking here
2478        // too keeps `configured_ids` a superset of the roster for any caller
2479        // that inserts a module directly.
2480        self.mark_configured(module.module_id());
2481        let mut modules = self
2482            .modules
2483            .lock()
2484            .unwrap_or_else(|poisoned| poisoned.into_inner());
2485        modules.insert(module.module_id().to_string(), module)
2486    }
2487
2488    /// Record that the supervisor has taken on `module_id`. Called before the
2489    /// module's first process is spawned, so that by the time that process can
2490    /// register, [`Self::is_configured`] already answers true.
2491    fn mark_configured(&self, module_id: &str) {
2492        self.configured_ids
2493            .lock()
2494            .unwrap_or_else(|poisoned| poisoned.into_inner())
2495            .insert(module_id.to_string());
2496    }
2497
2498    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2499    /// before it was ever put on the roster. A module already on the roster
2500    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2501    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502        let modules = self
2503            .modules
2504            .lock()
2505            .unwrap_or_else(|poisoned| poisoned.into_inner());
2506        if !modules.contains_key(module_id) {
2507            self.configured_ids
2508                .lock()
2509                .unwrap_or_else(|poisoned| poisoned.into_inner())
2510                .remove(module_id);
2511        }
2512    }
2513
2514    /// Whether `module_id` is a module this daemon supervises: on the roster,
2515    /// or about to be (its process is being spawned right now).
2516    ///
2517    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2518    /// for scopes: a supervised module's process can register and sync before
2519    /// [`Self::get`] can return it, and in that window it is still a module
2520    /// that will sync, not one that never will.
2521    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522        self.configured_ids
2523            .lock()
2524            .unwrap_or_else(|poisoned| poisoned.into_inner())
2525            .contains(module_id)
2526    }
2527
2528    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529        let modules = self
2530            .modules
2531            .lock()
2532            .unwrap_or_else(|poisoned| poisoned.into_inner());
2533        modules.get(module_id).cloned()
2534    }
2535
2536    pub(crate) fn record_late_health_answer(
2537        &self,
2538        module_id: &str,
2539        latency_ms: u64,
2540    ) -> Result<bool, SuperviseError> {
2541        let Some(module) = self.get(module_id) else {
2542            return Ok(false);
2543        };
2544        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546            state.health.last_late_answer_latency_ms = Some(latency_ms);
2547            // A late answer is an answer: the module served the probe, just past
2548            // the deadline. Leaving the miss streak in place while logging
2549            // "proves the module is alive" is how a CPU-starved module that
2550            // answers every probe a few seconds late still marches to the
2551            // threshold and gets killed — the exact kill class `NoAnswer` is
2552            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2553            // is degradation, and degradation reports; it does not restart.
2554            state.health.consecutive_failures = 0;
2555        })?;
2556        Ok(true)
2557    }
2558
2559    /// Arm the one-shot marker for the module process that this caller
2560    /// deliberately initiated severance against. Generic connection teardown
2561    /// must not call this:
2562    /// a surviving process would otherwise retain an exemption for a later
2563    /// genuine crash.
2564    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565        let Some(module) = self.get(module_id) else {
2566            return Ok(false);
2567        };
2568        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570            return Ok(false);
2571        };
2572        drop(snapshot);
2573        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574    }
2575
2576    pub fn list(&self) -> Vec<SupervisedModule> {
2577        let modules = self
2578            .modules
2579            .lock()
2580            .unwrap_or_else(|poisoned| poisoned.into_inner());
2581        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583        modules
2584    }
2585
2586    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587        self.activity_images
2588            .lock()
2589            .unwrap_or_else(|p| p.into_inner())
2590            .remove(module_id);
2591        self.spawn_nonces
2592            .lock()
2593            .unwrap_or_else(|poisoned| poisoned.into_inner())
2594            .remove(module_id);
2595        self.close_swap(module_id);
2596        let mut reserved_nonces = self
2597            .reserved_nonces
2598            .lock()
2599            .unwrap_or_else(|poisoned| poisoned.into_inner());
2600        if reserved_nonces.contains_key(module_id) {
2601            // The old nonce must die with the removed process, but the exact-id
2602            // gate remains until an operator explicitly releases it.
2603            reserved_nonces.insert(module_id.to_string(), None);
2604        }
2605        drop(reserved_nonces);
2606        self.reserved_prefix_owners
2607            .lock()
2608            .unwrap_or_else(|poisoned| poisoned.into_inner())
2609            .retain(|_, owner| owner != module_id);
2610        let removed = self
2611            .modules
2612            .lock()
2613            .unwrap_or_else(|poisoned| poisoned.into_inner())
2614            .remove(module_id);
2615        self.configured_ids
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        removed
2620    }
2621
2622    /// Remember a module removed by a non-preview rescan so route.open can
2623    /// distinguish that intentional removal from an unknown id.
2624    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625        self.removal_tombstones
2626            .lock()
2627            .unwrap_or_else(|poisoned| poisoned.into_inner())
2628            .insert(module_id.to_string(), unix_ms_now());
2629    }
2630
2631    /// Return how long ago a rescan removed this module in milliseconds.
2632    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633        self.removal_tombstones
2634            .lock()
2635            .unwrap_or_else(|poisoned| poisoned.into_inner())
2636            .get(module_id)
2637            .copied()
2638            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639    }
2640
2641    /// Retire a reserved-id gate only after its module has left supervision.
2642    ///
2643    /// A retained gate has no live nonce (`None`), so releasing any other entry
2644    /// would weaken a currently configured or otherwise active reservation.
2645    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646        if self.get(module_id).is_some() {
2647            return false;
2648        }
2649        let mut reserved_nonces = self
2650            .reserved_nonces
2651            .lock()
2652            .unwrap_or_else(|poisoned| poisoned.into_inner());
2653        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654            return false;
2655        }
2656        reserved_nonces.remove(module_id);
2657        true
2658    }
2659
2660    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661        Arc::clone(&self.operation_lock)
2662    }
2663}
2664
2665/// Process supervisor for subc-owned singleton modules.
2666#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668    registry: Arc<Registry>,
2669    restart_policy: RestartPolicy,
2670    drain_timeout: Duration,
2671    connection_file_path: Option<PathBuf>,
2672    capture_logs_dir: Option<PathBuf>,
2673    forwarding: Option<Arc<ForwardingTable>>,
2674    process_liveness: Arc<SupervisorProcessLiveness>,
2675    supervisor_handle: Option<SupervisorHandle>,
2676    health: HealthConfig,
2677    daemon_start_clock: crate::clock::StartClock,
2678    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679    spawn_events: SpawnEventFeed,
2680    provenance_probe: ExecutableIdentityProbe,
2681    /// Every process spawned through this supervisor (and its clones) and not
2682    /// yet reaped, so daemon shutdown can end them.
2683    child_roster: ChildRoster,
2684    #[cfg(target_os = "linux")]
2685    cgroup_placement: Option<subc_cgroup::Placement>,
2686    #[cfg(test)]
2687    test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690/// A roster entry whose monitor and first process have not started yet.
2691/// Keeping the command receiver here lets discovery serve status immediately
2692/// while start/stop requests wait until bootstrap has published the connection file.
2693pub(crate) struct RegisteredConfiguredModule {
2694    spec: ModuleSpec,
2695    runtime: SupervisorRuntimeConfig,
2696    module: SupervisedModule,
2697    commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700/// Test-only hook run on the path that takes on a new module, right after its
2701/// first `spawn_child` returns (the process exists and could already be
2702/// registering) and before that process is handed to the module's supervise
2703/// loop and put on the roster. Lets a test observe what a fast child would see
2704/// in that window without racing a real one.
2705#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715        f.write_str("AfterFirstSpawnHook")
2716    }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721    fn run(&self, module_id: &str) {
2722        if let Some(hook) = &self.0 {
2723            hook(module_id);
2724        }
2725    }
2726}
2727
2728impl Supervisor {
2729    #[cfg(test)]
2730    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731        let supervisor = Self::new(registry, policy);
2732        #[cfg(target_os = "macos")]
2733        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734        supervisor
2735    }
2736    /// Verify the trampoline once when configured. Missing private OS support
2737    /// refuses every macOS launch by name but does not stop the daemon's control
2738    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2739    /// entry point; the library must not exec an arbitrary hosting program.
2740    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741        let path = path.into();
2742        #[cfg(target_os = "macos")]
2743        {
2744            let result = probe_privacy_trampoline(&path).map(|()| path);
2745            if let Err(cause) = &result {
2746                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747            }
2748            self.child_roster.set_privacy_trampoline(result);
2749        }
2750        #[cfg(not(target_os = "macos"))]
2751        let _ = path;
2752        self
2753    }
2754    /// The first step of an announced daemon shutdown, before the notice and
2755    /// before any connection is closed.
2756    ///
2757    /// Sets the daemon-shutdown flag first: from here on no module is
2758    /// respawned (crash restart, operator restart, or swap), and every child
2759    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2760    /// the module exits on the EOF this shutdown gives it or is signalled by a
2761    /// service manager that kills the whole cgroup. Then writes the journal's
2762    /// shutdown marker, which records the instant and closes this daemon
2763    /// incarnation's stretch of the journal.
2764    #[cfg(unix)]
2765    pub(crate) fn begin_daemon_shutdown(&self) {
2766        self.child_roster.close();
2767        if let Some(journal) = &self.terminal_journal {
2768            journal.stamp_shutdown();
2769        }
2770    }
2771
2772    /// Announce a cut while established connections can still carry replies.
2773    /// These budgets promise notice and a bounded wait, not child completion;
2774    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2775    #[cfg(unix)]
2776    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2777        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2778        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2779        let Some(forwarding) = &self.forwarding else {
2780            return Ok(());
2781        };
2782        let modules = forwarding
2783            .begin_daemon_drain()
2784            .map_err(SuperviseError::Forwarding)?;
2785        let deadline_ms =
2786            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2787        let mut notices = tokio::task::JoinSet::new();
2788        let mut drains = Vec::new();
2789        for (module_id, endpoint) in modules {
2790            let Some(target) = forwarding
2791                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2792                .map_err(SuperviseError::Forwarding)?
2793            else {
2794                continue;
2795            };
2796            let routes = forwarding
2797                .endpoint_routes(target.endpoint)
2798                .map_err(SuperviseError::Forwarding)?;
2799            // Restart allows deployed consumers to reopen after the new daemon
2800            // appears. The wire reason stays `restart`; what tells a daemon cut
2801            // apart from a module restart afterwards is the terminal record
2802            // itself, whose disposition is `daemon_shutdown` for every exit
2803            // observed once `begin_daemon_shutdown` has run.
2804            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2805                reason: RouteCloseReason::Restart,
2806                deadline_ms,
2807            })
2808            .expect("module draining serializes");
2809            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2810            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2811            for route in routes {
2812                let client = route.goodbye_target;
2813                if let Some((_, channels)) = clients
2814                    .iter_mut()
2815                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2816                {
2817                    channels.push(client.channel);
2818                } else {
2819                    let channel = client.channel;
2820                    clients.push((client, vec![channel]));
2821                }
2822            }
2823            for (client, mut channels) in clients {
2824                channels.sort_unstable();
2825                channels.dedup();
2826                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2827                    module_id: module_id.clone(),
2828                    channels,
2829                    reason: RouteCloseReason::Restart,
2830                })
2831                .expect("route closing serializes");
2832                recipients.push((client.sink, client.negotiated_ver, closing));
2833            }
2834            for (sink, version, body) in recipients {
2835                notices.spawn(async move {
2836                    let frame = Frame::build_with_version(
2837                        version,
2838                        FrameType::Push,
2839                        control_flags(),
2840                        0,
2841                        0,
2842                        0,
2843                        body,
2844                    )
2845                    .expect("bounded lifecycle notice frame builds");
2846                    sink.send_flushed(frame).await
2847                });
2848            }
2849            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2850            drains.push((module_id, target.endpoint, gauges));
2851        }
2852        // A quiet forwarding table is not proof that queued notices reached the
2853        // socket. Wait for writer flush acknowledgements before testing quiescence.
2854        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2855        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2856            if !matches!(result, Ok(Ok(()))) {
2857                warn!(?result, "daemon shutdown notice delivery failed");
2858            }
2859        }
2860        notices.abort_all();
2861        let deadline = Instant::now() + DRAIN_BUDGET;
2862        let mut waits = tokio::task::JoinSet::new();
2863        for (module_id, endpoint, gauges) in drains {
2864            let forwarding = Arc::clone(forwarding);
2865            let mut runtime = self.runtime_config();
2866            runtime.health.cadence = Duration::from_millis(100);
2867            waits.spawn(async move {
2868                let wait_result = wait_for_forwarding_quiescence(
2869                    &forwarding,
2870                    &module_id,
2871                    &runtime,
2872                    endpoint,
2873                    deadline,
2874                    &gauges,
2875                    DrainScope::Endpoint(endpoint),
2876                )
2877                .await;
2878                // Store the measurement before the task returns: quiescence
2879                // may be reached just as the deadline stops the join loop from
2880                // collecting results. Unfinished or failed waits report false.
2881                forwarding
2882                    .record_daemon_drain_result(
2883                        endpoint,
2884                        drained_after_quiescence_wait(&wait_result),
2885                    )
2886                    .map_err(SuperviseError::Forwarding)?;
2887                wait_result
2888            });
2889        }
2890        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2891            if !matches!(result, Ok(Ok(true))) {
2892                warn!(?result, "daemon shutdown drain did not reach quiescence");
2893            }
2894        }
2895        Ok(())
2896    }
2897
2898    /// The last step of an announced daemon shutdown, after the notice and the
2899    /// drain: send every registered module a module GOODBYE, the same planned
2900    /// stop signal `ck module stop` gives, then close every connection so each
2901    /// subc module sees EOF and starts its own teardown, then end every
2902    /// supervised child that has not exited
2903    /// by its own deadline (its drain budget, capped). Modules lead their own
2904    /// process groups, so a
2905    /// service manager's group kill no longer reaches them; without this a
2906    /// child that does not stop on EOF (every `protocol: "none"` child, which
2907    /// has no connection) would outlive the daemon. Every wait is bounded (see
2908    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2909    #[cfg(unix)]
2910    pub(crate) async fn end_children_for_daemon_shutdown(
2911        &self,
2912        already_escalated: bool,
2913        escalate: impl std::future::Future<Output = ()>,
2914    ) {
2915        tokio::pin!(escalate);
2916        let mut escalated = already_escalated;
2917        if let Some(forwarding) = &self.forwarding {
2918            let reason = CloseReason::new(
2919                "daemon_shutdown",
2920                "the daemon is exiting after its shutdown notice and drain",
2921            );
2922            if escalated {
2923                // The operator asked to stop waiting: queue the GOODBYEs but
2924                // do not wait for them to be written.
2925                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2926            } else {
2927                tokio::select! {
2928                    biased;
2929                    _ = escalate.as_mut() => {
2930                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2931                        escalated = true;
2932                    }
2933                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2934                }
2935            }
2936            let closed = forwarding.close_all_connections(&reason);
2937            debug!(closed, "closed established connections for daemon shutdown");
2938        }
2939        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2940        // already completed and must not be polled again; the child shutdown
2941        // wait is told it is escalated and gets a future that never fires.
2942        let escalated_here = escalated && !already_escalated;
2943        let remaining_escalate = async move {
2944            if escalated_here {
2945                std::future::pending::<()>().await;
2946            } else {
2947                escalate.await;
2948            }
2949        };
2950        crate::child_roster::end_children_for_daemon_shutdown(
2951            &self.child_roster,
2952            escalated,
2953            remaining_escalate,
2954        )
2955        .await;
2956    }
2957
2958    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2959        Self {
2960            registry,
2961            restart_policy,
2962            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2963            connection_file_path: None,
2964            capture_logs_dir: None,
2965            forwarding: None,
2966            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2967            supervisor_handle: None,
2968            health: HealthConfig::default(),
2969            daemon_start_clock: crate::clock::StartClock::capture(),
2970            terminal_journal: None,
2971            spawn_events: SpawnEventFeed::default(),
2972            provenance_probe: ExecutableIdentityProbe::default(),
2973            child_roster: ChildRoster::default(),
2974            #[cfg(target_os = "linux")]
2975            cgroup_placement: None,
2976            #[cfg(test)]
2977            test_after_first_spawn: AfterFirstSpawnHook::default(),
2978        }
2979    }
2980
2981    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2982        self.drain_timeout = drain_timeout;
2983        self
2984    }
2985
2986    pub fn with_process_liveness(
2987        mut self,
2988        process_liveness: Arc<SupervisorProcessLiveness>,
2989    ) -> Self {
2990        self.process_liveness = process_liveness;
2991        self
2992    }
2993
2994    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2995        self.connection_file_path = Some(connection_file_path.into());
2996        self
2997    }
2998
2999    /// Enables daemon-owned capture files for supervised stdout and stderr.
3000    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3001        self.capture_logs_dir = Some(logs_dir.into());
3002        self
3003    }
3004
3005    /// Names this daemon lifetime in spawn events, independently of whether a
3006    /// terminal journal is configured.
3007    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3008        // A millisecond start stamp can repeat after clock rollback or a rapid
3009        // restart. Use the connection file's random daemon_id instead: it already
3010        // identifies this daemon lifetime independently of the wall clock.
3011        self.spawn_events.configure_incarnation(daemon_incarnation);
3012        self
3013    }
3014
3015    /// Enables best-effort history shared by every supervised module. Without
3016    /// it, terminal history is kept only in each module's in-memory ring.
3017    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3018        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3019        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3020            path,
3021            daemon_incarnation,
3022        )));
3023        this
3024    }
3025
3026    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3027        self.forwarding = Some(forwarding);
3028        self
3029    }
3030
3031    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3032        self.spawn_events = supervisor_handle.spawn_events.clone();
3033        self.supervisor_handle = Some(supervisor_handle);
3034        self
3035    }
3036
3037    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3038        self.health = health;
3039        self
3040    }
3041
3042    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3043    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3044    /// record is kept.
3045    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3046        self.child_roster.record_to(path.into());
3047        self
3048    }
3049
3050    #[cfg(target_os = "linux")]
3051    pub fn with_cgroup_placement(
3052        mut self,
3053        cgroup_placement: Option<subc_cgroup::Placement>,
3054    ) -> Self {
3055        self.cgroup_placement = cgroup_placement;
3056        self
3057    }
3058
3059    /// Spawn `spec.program` and start monitoring it.
3060    ///
3061    /// The child is expected to parse `--subc <connection-file-path>`, read the
3062    /// TCP+key connection file, authenticate to the already-running listener, and
3063    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3064    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3065        validate_spec(&spec)?;
3066        self.establish_identity(&spec);
3067
3068        let runtime = self.runtime_config();
3069        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3070        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3071        let spawned = spawn_child(
3072            &spec,
3073            runtime.connection_file_path.as_deref(),
3074            self.supervisor_handle.as_ref(),
3075            &runtime.stderr_ring,
3076            runtime.capture_logs_dir.as_deref(),
3077            &runtime.child_roster,
3078            #[cfg(target_os = "linux")]
3079            runtime.cgroup_placement.as_ref(),
3080        );
3081        #[cfg(test)]
3082        self.test_after_first_spawn.run(&spec.module_id);
3083        let child = match spawned {
3084            Ok(child) => child,
3085            Err(err) => {
3086                // Unlike the configured paths, a failed `spawn` leaves nothing
3087                // on the roster, so the module must not stay marked configured.
3088                self.abandon_unrostered(&spec.module_id);
3089                return Err(err);
3090            }
3091        };
3092        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3093        if let Some(spawn) = activity_spawn {
3094            spawn.complete(&snapshot);
3095        }
3096
3097        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3098    }
3099
3100    /// Make `spec`'s module count as configured, with its identity gates
3101    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3102    /// exists.
3103    ///
3104    /// Every path that takes on a new module calls this before `spawn_child`.
3105    /// The order is the point: the child can connect, register, sync its
3106    /// scopes and ask about them as soon as it is spawned, and the module is
3107    /// only put on the roster after `spawn_child` returns. Were the mark set
3108    /// with the roster entry, a fast child would see its own owner reported
3109    /// as not configured, and a scoped `route.open` in that window would be
3110    /// refused as terminal `scope_not_live` ("will never sync") instead of
3111    /// retryable `scope_not_synced`.
3112    fn establish_identity(&self, spec: &ModuleSpec) {
3113        if let Some(supervisor_handle) = &self.supervisor_handle {
3114            supervisor_handle.apply_identity_configuration(spec);
3115            supervisor_handle.mark_configured(&spec.module_id);
3116        }
3117    }
3118
3119    /// Take back [`Self::establish_identity`]'s configured mark when the
3120    /// module will not be put on the roster after all.
3121    fn abandon_unrostered(&self, module_id: &str) {
3122        if let Some(supervisor_handle) = &self.supervisor_handle {
3123            supervisor_handle.unmark_configured_unless_rostered(module_id);
3124        }
3125    }
3126
3127    /// Record a freshly spawned first process as running. On failure the
3128    /// module never reaches the roster, so its configured mark is taken back.
3129    fn mark_first_process_running(
3130        &self,
3131        spec: &ModuleSpec,
3132        runtime: &SupervisorRuntimeConfig,
3133        snapshot: &SharedSnapshot,
3134        child: &SupervisedChild,
3135    ) -> Result<(), SuperviseError> {
3136        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3137            self.abandon_unrostered(&spec.module_id);
3138            return Err(err);
3139        }
3140        self.process_liveness
3141            .track(spec.module_id.clone(), Arc::clone(snapshot));
3142        Ok(())
3143    }
3144
3145    /// Start supervising a module declared in daemon configuration.
3146    ///
3147    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3148    /// failures in the supervisor handle so operator-facing `supervisor.list`
3149    /// reflects every configured module while daemon startup continues.
3150    pub fn supervise_configured(
3151        &self,
3152        spec: ModuleSpec,
3153        enabled: bool,
3154    ) -> Result<SupervisedModule, SuperviseError> {
3155        self.supervise_configured_requested(
3156            spec,
3157            enabled,
3158            self.health.clone(),
3159            None,
3160            self.restart_policy,
3161            None,
3162        )
3163    }
3164
3165    /// Supervise a configured module with its own health, drain, and crash
3166    /// budget. The restart policy is per-module because the config file is:
3167    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3168    /// module that is expensive to restart should not be forced onto the same
3169    /// budget as one that is cheap.
3170    pub fn supervise_configured_with_health(
3171        &self,
3172        spec: ModuleSpec,
3173        enabled: bool,
3174        health: HealthConfig,
3175        drain_timeout_ms: Option<u64>,
3176        restart_policy: RestartPolicy,
3177    ) -> Result<SupervisedModule, SuperviseError> {
3178        self.supervise_configured_requested(
3179            spec,
3180            enabled,
3181            health,
3182            drain_timeout_ms,
3183            restart_policy,
3184            None,
3185        )
3186    }
3187
3188    pub(crate) fn supervise_configured_requested(
3189        &self,
3190        spec: ModuleSpec,
3191        enabled: bool,
3192        health: HealthConfig,
3193        drain_timeout_ms: Option<u64>,
3194        restart_policy: RestartPolicy,
3195        requester: Option<Principal>,
3196    ) -> Result<SupervisedModule, SuperviseError> {
3197        let registered = self.register_configured_with_health(
3198            spec,
3199            enabled,
3200            health,
3201            drain_timeout_ms,
3202            restart_policy,
3203        )?;
3204        self.start_registered_configured(registered, requester)
3205    }
3206
3207    /// Record a configured module as starting or disabled, without launching a
3208    /// process or running its monitor. Bootstrap registers the entire roster
3209    /// before publishing discovery, then activates it once children can read
3210    /// the connection file passed in their `--subc` argument.
3211    pub(crate) fn register_configured_with_health(
3212        &self,
3213        spec: ModuleSpec,
3214        enabled: bool,
3215        health: HealthConfig,
3216        drain_timeout_ms: Option<u64>,
3217        restart_policy: RestartPolicy,
3218    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3219        validate_spec(&spec)?;
3220        self.establish_identity(&spec);
3221
3222        let mut runtime = self.runtime_config();
3223        runtime.health = health;
3224        runtime.restart_policy = restart_policy;
3225        if let Some(ms) = drain_timeout_ms {
3226            runtime.drain_timeout = Duration::from_millis(ms);
3227            *runtime
3228                .effective_drain_timeout
3229                .lock()
3230                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3231        }
3232        let snapshot = if enabled {
3233            SupervisorSnapshot::starting()
3234        } else {
3235            SupervisorSnapshot::disabled()
3236        };
3237        let snapshot = {
3238            let mut snapshot = snapshot;
3239            if enabled {
3240                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3241                {
3242                    snapshot.waiting_for_previous_pid = Some(pid);
3243                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3244                }
3245            }
3246            snapshot
3247        };
3248        let (module, commands) =
3249            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3250        Ok(RegisteredConfiguredModule {
3251            spec,
3252            runtime,
3253            module,
3254            commands,
3255        })
3256    }
3257
3258    /// Activate a previously registered module. Queued control requests are
3259    /// served by the monitor after the initial configured spawn has settled.
3260    pub(crate) fn start_registered_configured(
3261        &self,
3262        registered: RegisteredConfiguredModule,
3263        requester: Option<Principal>,
3264    ) -> Result<SupervisedModule, SuperviseError> {
3265        let RegisteredConfiguredModule {
3266            spec,
3267            mut runtime,
3268            module,
3269            commands,
3270        } = registered;
3271        runtime.start_requester = requester;
3272        let snapshot = &module.inner.snapshot;
3273        let should_spawn = {
3274            let state = lock_snapshot(snapshot)?;
3275            state.enabled && state.waiting_for_previous_pid.is_none()
3276        };
3277        let child = if should_spawn {
3278            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3279            let spawned = spawn_child(
3280                &spec,
3281                runtime.connection_file_path.as_deref(),
3282                self.supervisor_handle.as_ref(),
3283                &runtime.stderr_ring,
3284                runtime.capture_logs_dir.as_deref(),
3285                &runtime.child_roster,
3286                #[cfg(target_os = "linux")]
3287                runtime.cgroup_placement.as_ref(),
3288            );
3289            #[cfg(test)]
3290            self.test_after_first_spawn.run(&spec.module_id);
3291            match spawned {
3292                Ok(child) => {
3293                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3294                    if let Some(spawn) = activity_spawn {
3295                        spawn.complete(snapshot);
3296                    }
3297                    Some(child)
3298                }
3299                Err(err) => {
3300                    if runtime.health.critical {
3301                        error!(
3302                            module_id = %spec.module_id,
3303                            program = %spec.program.display(),
3304                            error = %err,
3305                            "critical configured module failed to spawn; marking failed and alerting"
3306                        );
3307                    } else {
3308                        error!(
3309                            module_id = %spec.module_id,
3310                            program = %spec.program.display(),
3311                            error = %err,
3312                            "configured module failed to spawn; marking failed and continuing"
3313                        );
3314                    }
3315                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3316                        state.state = ModuleState::Failed;
3317                        clear_current_process_facts(state);
3318                    })?;
3319                    None
3320                }
3321            }
3322        } else {
3323            None
3324        };
3325        self.start_monitor(spec, runtime, &module, child, commands);
3326        Ok(module)
3327    }
3328
3329    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3330        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3331        SupervisorRuntimeConfig {
3332            start_requester: None,
3333            pending_activity_restart: false,
3334            provenance_probe: self.provenance_probe.clone(),
3335            scheduled_respawn: Arc::default(),
3336            deferred_reload_reply: Arc::default(),
3337            restart_policy: self.restart_policy,
3338            drain_timeout: self.drain_timeout,
3339            // Shared with this module's roster copy: daemon shutdown waits on
3340            // each child for the module's own drain budget, as resolved now.
3341            child_roster: self
3342                .child_roster
3343                .for_module(Arc::clone(&effective_drain_timeout)),
3344            effective_drain_timeout,
3345            default_drain_timeout: self.drain_timeout,
3346            health: self.health.clone(),
3347            connection_file_path: self.connection_file_path.clone(),
3348            capture_logs_dir: self.capture_logs_dir.clone(),
3349            forwarding: self.forwarding.clone(),
3350            supervisor_handle: self.supervisor_handle.clone(),
3351            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3352            terminal_ring: Arc::new(Mutex::new(
3353                TerminalRing::new(
3354                    TerminalRingConfig::default(),
3355                    self.daemon_start_clock.started_at_ms(),
3356                )
3357                .with_start_clock(self.daemon_start_clock)
3358                .with_journal(self.terminal_journal.clone())
3359                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3360            )),
3361            spawn_events: self.spawn_events.clone(),
3362            #[cfg(target_os = "linux")]
3363            cgroup_placement: self.cgroup_placement.clone(),
3364            #[cfg(test)]
3365            test_seed_stale_facts_before_enable_spawn: false,
3366            #[cfg(test)]
3367            test_reload_exit_record_gate: None,
3368        }
3369    }
3370
3371    fn supervised_module(
3372        &self,
3373        spec: ModuleSpec,
3374        runtime: SupervisorRuntimeConfig,
3375        snapshot: SharedSnapshot,
3376        child: Option<SupervisedChild>,
3377    ) -> SupervisedModule {
3378        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3379        self.start_monitor(spec, runtime, &module, child, commands);
3380        module
3381    }
3382
3383    fn register_module(
3384        &self,
3385        spec: &ModuleSpec,
3386        runtime: &SupervisorRuntimeConfig,
3387        snapshot: SharedSnapshot,
3388    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3389        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3390            spec: spec.clone(),
3391            health: runtime.health.clone(),
3392        }));
3393        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3394        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3395        // The module's OWN policy, which may be its per-module config rather than
3396        // the supervisor-wide one; status must report the budget the supervise
3397        // loop actually enforces.
3398        let restart_policy = runtime.restart_policy;
3399        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3400        let (tx, rx) = mpsc::channel(4);
3401        let module_id = spec.module_id.clone();
3402        let module = SupervisedModule {
3403            inner: Arc::new(SupervisedModuleInner {
3404                module_id: module_id.clone(),
3405                registry: Arc::clone(&self.registry),
3406                snapshot,
3407                configuration,
3408                stderr_ring,
3409                terminal_ring,
3410                commands: tx,
3411                monitor: Mutex::new(None),
3412                restart_policy,
3413                effective_drain_timeout,
3414                provenance_probe: self.provenance_probe.clone(),
3415            }),
3416        };
3417        // Identity gates and the configured mark must precede the roster entry,
3418        // so a client never sees a module without its admission policy.
3419        if let Some(supervisor_handle) = &self.supervisor_handle {
3420            supervisor_handle.insert(module.clone());
3421        }
3422        (module, rx)
3423    }
3424
3425    fn start_monitor(
3426        &self,
3427        spec: ModuleSpec,
3428        runtime: SupervisorRuntimeConfig,
3429        module: &SupervisedModule,
3430        child: Option<SupervisedChild>,
3431        commands: mpsc::Receiver<SupervisorCommand>,
3432    ) {
3433        let monitor = tokio::spawn(supervise_loop(
3434            spec,
3435            runtime,
3436            Arc::clone(&self.registry),
3437            Arc::clone(&self.process_liveness),
3438            Arc::clone(&module.inner.snapshot),
3439            child,
3440            commands,
3441        ));
3442        *module
3443            .inner
3444            .monitor
3445            .lock()
3446            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3447    }
3448}
3449
3450impl Default for Supervisor {
3451    fn default() -> Self {
3452        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3453    }
3454}
3455
3456/// Handle to one supervised child process.
3457#[derive(Clone)]
3458pub struct SupervisedModule {
3459    inner: Arc<SupervisedModuleInner>,
3460}
3461
3462struct SupervisedModuleInner {
3463    module_id: String,
3464    registry: Arc<Registry>,
3465    snapshot: SharedSnapshot,
3466    configuration: Arc<Mutex<SupervisedConfiguration>>,
3467    stderr_ring: Arc<Mutex<StderrRing>>,
3468    terminal_ring: Arc<Mutex<TerminalRing>>,
3469    commands: mpsc::Sender<SupervisorCommand>,
3470    monitor: Mutex<Option<JoinHandle<()>>>,
3471    /// Copied from the supervisor's runtime config at spawn so `status()` can
3472    /// report the restart budget without reaching back into the supervisor. The
3473    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3474    restart_policy: RestartPolicy,
3475    effective_drain_timeout: Arc<Mutex<Duration>>,
3476    provenance_probe: ExecutableIdentityProbe,
3477}
3478
3479impl fmt::Debug for SupervisedModule {
3480    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3481        f.debug_struct("SupervisedModule")
3482            .field("module_id", &self.inner.module_id)
3483            .field("status", &self.status())
3484            .finish_non_exhaustive()
3485    }
3486}
3487
3488impl SupervisedModule {
3489    pub fn module_id(&self) -> &str {
3490        &self.inner.module_id
3491    }
3492
3493    /// Test-only: put one probe miss on the streak, the way
3494    /// `handle_health_probe_failure` does, so tests can assert what a later
3495    /// event does to the streak without driving the whole probe loop.
3496    #[cfg(test)]
3497    pub(crate) fn record_health_probe_failure_for_test(
3498        &self,
3499        detail: &str,
3500    ) -> Result<(), SuperviseError> {
3501        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3502            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3503            state.health.detail = Some(detail.to_string());
3504        })
3505    }
3506
3507    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3508        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3509    }
3510
3511    pub(crate) fn registration_end_reason(
3512        &self,
3513    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3514        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3515        Ok(match snapshot.state {
3516            ModuleState::Draining if snapshot.draining_to_replace => {
3517                Some(RegistrationEndReason::SupervisorRestart)
3518            }
3519            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3520            ModuleState::Restarting if snapshot.draining_to_replace => {
3521                Some(RegistrationEndReason::SupervisorRestart)
3522            }
3523            _ => None,
3524        })
3525    }
3526
3527    /// The module's retained stderr, newest lines last.
3528    ///
3529    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3530    /// module, `supervisor.list` renders every module, and putting it in the
3531    /// shared snapshot would make each status read carry a payload almost nobody
3532    /// asked for. Callers that want the text ask for it.
3533    pub fn stderr_tail(
3534        &self,
3535        max_lines: Option<usize>,
3536        max_bytes: Option<usize>,
3537    ) -> StderrTailSnapshot {
3538        self.inner
3539            .stderr_ring
3540            .lock()
3541            .unwrap_or_else(|poisoned| poisoned.into_inner())
3542            .snapshot(max_lines, max_bytes)
3543    }
3544
3545    /// The module's bounded terminal history, oldest retained exit first.
3546    ///
3547    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3548    /// daemon whose in-memory history was necessarily reset.
3549    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3550        self.inner
3551            .terminal_ring
3552            .lock()
3553            .unwrap_or_else(|poisoned| poisoned.into_inner())
3554            .snapshot()
3555    }
3556
3557    /// Retained observations from the current ring and all journal generations.
3558    ///
3559    /// Blocking: this reads the journal files. Async callers use
3560    /// [`Self::read_durable_terminal_history`].
3561    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3562        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3563    }
3564
3565    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3566    /// read (up to every retained generation) never occupies a runtime worker.
3567    /// Fails only if the blocking task could not finish (runtime shutdown or a
3568    /// panic in the read).
3569    pub(crate) async fn read_durable_terminal_history(
3570        &self,
3571    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3572        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3573        let module_id = self.inner.module_id.clone();
3574        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3575            .await
3576    }
3577
3578    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3579        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3580            .map(|(status, _)| status)
3581    }
3582
3583    pub(crate) fn record_deliberate_severance(
3584        &self,
3585        identity: ProcessIdentity,
3586    ) -> Result<bool, SuperviseError> {
3587        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3588        if snapshot.pid != Some(identity.pid)
3589            || snapshot.process_start_time != Some(identity.start_time)
3590        {
3591            return Ok(false);
3592        }
3593        snapshot.deliberate_severance = Some(identity);
3594        Ok(true)
3595    }
3596
3597    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3598    ///
3599    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3600    /// does not produce reader-observability logs.
3601    pub(crate) fn status_for_control(
3602        &self,
3603        caller: &'static str,
3604    ) -> Result<ModuleStatus, SuperviseError> {
3605        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3606            .map(|(status, _)| status)
3607    }
3608
3609    fn status_with_snapshot_lock(
3610        &self,
3611        snapshot: &SharedSnapshot,
3612        caller: Option<&'static str>,
3613    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3614        let mut guard = match caller {
3615            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3616            None => lock_snapshot(snapshot)?,
3617        };
3618        // Read the budget through the pruning path so a reader sees the same
3619        // in-window count the restart decision would use, not a stale total.
3620        let restart_count =
3621            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3622        let snapshot = guard.clone();
3623        drop(guard);
3624        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3625            SuperviseError::StatePoisoned {
3626                module_id: Some(self.inner.module_id.clone()),
3627            }
3628        })?;
3629        let registration_active = self
3630            .inner
3631            .registry
3632            .get_module(&self.inner.module_id)
3633            .map_err(SuperviseError::Registry)?
3634            .is_some();
3635        let protocol = snapshot
3636            .spawned_protocol
3637            .unwrap_or(self.declared_protocol()?);
3638        let running_process =
3639            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3640        // Registration is the difference between the two protocols and the only
3641        // one: a subc module that has not registered cannot serve a request even
3642        // though its process is up, and a `none` module never registers at all,
3643        // so requiring it there would pin `live` to false for the whole life of
3644        // a perfectly healthy process.
3645        let live = match protocol {
3646            ModuleProtocol::Subc => running_process && registration_active,
3647            ModuleProtocol::None => running_process,
3648        };
3649
3650        Ok((
3651            ModuleStatus {
3652                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3653                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3654                module_id: self.inner.module_id.clone(),
3655                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3656                state: snapshot.state,
3657                enabled: snapshot.enabled,
3658                process_alive: snapshot.process_alive,
3659                registration_active,
3660                protocol,
3661                live,
3662                restart_count,
3663                lifetime_restarts: snapshot.lifetime_restarts,
3664                spawn_generation: snapshot.spawn_generation,
3665                max_restarts: self.inner.restart_policy.max_restarts,
3666                restart_window: self.inner.restart_policy.window,
3667                drain_timeout,
3668                restart_backoff: self.inner.restart_policy.backoff,
3669                restart_max_backoff: self.inner.restart_policy.max_backoff,
3670                pid: snapshot.reported_pid(),
3671                spawned_at_ms: snapshot.spawned_at_ms,
3672                spawned_from: snapshot.spawned_from,
3673                launch_spec_digest: snapshot.launch_spec_digest,
3674                process_start_time: snapshot.process_start_time,
3675                last_exit: snapshot.last_exit,
3676                health: snapshot.health,
3677            },
3678            snapshot.spawned_file_identity.clone(),
3679        ))
3680    }
3681
3682    #[cfg(test)]
3683    pub(crate) fn hold_snapshot_for_test(
3684        &self,
3685        acquired: std::sync::mpsc::Sender<()>,
3686        hold: Duration,
3687    ) -> std::thread::JoinHandle<()> {
3688        let snapshot = Arc::clone(&self.inner.snapshot);
3689        std::thread::spawn(move || {
3690            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3691            acquired
3692                .send(())
3693                .expect("test receiver waits for snapshot lock");
3694            std::thread::sleep(hold);
3695        })
3696    }
3697
3698    /// The status and the running-image check for `supervisor.provenance`,
3699    /// taken from one status read. The exec acknowledgement can land between
3700    /// two separate reads, and the reply would then pair "no pid yet" with an
3701    /// image observed after the module started, which describes no single
3702    /// moment.
3703    pub(crate) async fn status_and_running_image_agreement(
3704        &self,
3705    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3706        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3707        let image = self
3708            .inner
3709            .provenance_probe
3710            .observe(
3711                status.pid,
3712                status.spawned_from.as_deref(),
3713                identity,
3714                status.process_start_time,
3715            )
3716            .await;
3717        Ok((status, image))
3718    }
3719
3720    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3721        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3722            Ok(snapshot) => snapshot.clone(),
3723            Err(_) => {
3724                return subc_control::RunningImageAgreement::Unavailable {
3725                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3726                };
3727            }
3728        };
3729        self.inner
3730            .provenance_probe
3731            .observe(
3732                snapshot.reported_pid(),
3733                snapshot.spawned_from.as_deref(),
3734                snapshot.spawned_file_identity.clone(),
3735                snapshot.process_start_time,
3736            )
3737            .await
3738    }
3739
3740    /// Memory and CPU time of the module's current process, read now. Only the
3741    /// process the supervisor spawned is read, not processes it has started.
3742    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3743        #[cfg(windows)]
3744        {
3745            let image = lock_snapshot(&self.inner.snapshot)
3746                .ok()
3747                .and_then(|snapshot| {
3748                    snapshot
3749                        .spawned_file_identity
3750                        .as_ref()
3751                        .and_then(|identity| identity.windows_image.clone())
3752                });
3753            match image {
3754                Some(image) => {
3755                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3756                }
3757                None => subc_control::ChildResourceUsage::Unavailable {
3758                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3759                },
3760            }
3761        }
3762        #[cfg(not(windows))]
3763        {
3764            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3765                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3766                Err(_) => {
3767                    return subc_control::ChildResourceUsage::Unavailable {
3768                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3769                    }
3770                }
3771            };
3772            crate::child_resources::read(pid, start_time)
3773        }
3774    }
3775
3776    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3777        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3778        Ok(match snapshot.state {
3779            ModuleState::Restarting => true,
3780            ModuleState::Failed | ModuleState::Disabled => false,
3781            _ if snapshot.enabled && snapshot.respawn_pending => true,
3782            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3783        })
3784    }
3785
3786    #[cfg(test)]
3787    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3788        self.is_warming_with_snapshot_lock(None)
3789    }
3790
3791    pub(crate) fn is_warming_for_control(
3792        &self,
3793        caller: &'static str,
3794    ) -> Result<bool, SuperviseError> {
3795        self.is_warming_with_snapshot_lock(Some(caller))
3796    }
3797
3798    fn is_warming_with_snapshot_lock(
3799        &self,
3800        caller: Option<&'static str>,
3801    ) -> Result<bool, SuperviseError> {
3802        let snapshot = match caller {
3803            Some(caller) => {
3804                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3805            }
3806            None => lock_snapshot(&self.inner.snapshot)?,
3807        }
3808        .clone();
3809        Ok(matches!(
3810            snapshot.state,
3811            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3812        ))
3813    }
3814
3815    /// Drain the module and stop monitoring it.
3816    pub async fn drain(&self) -> Result<(), SuperviseError> {
3817        self.stop().await
3818    }
3819
3820    pub(crate) async fn retire_requested(
3821        &self,
3822        requester: Option<Principal>,
3823    ) -> Result<(), SuperviseError> {
3824        match self.state()? {
3825            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3826            ModuleState::Starting
3827            | ModuleState::Running
3828            | ModuleState::Unresponsive
3829            | ModuleState::Restarting
3830            | ModuleState::Draining
3831            | ModuleState::Disabled => {}
3832        }
3833
3834        let (reply_tx, reply_rx) = oneshot::channel();
3835        self.inner
3836            .commands
3837            .send(SupervisorCommand::Retire {
3838                requester,
3839                reply: reply_tx,
3840            })
3841            .await
3842            .map_err(|_| SuperviseError::CommandClosed {
3843                module_id: self.inner.module_id.clone(),
3844            })?;
3845        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3846            module_id: self.inner.module_id.clone(),
3847        })?
3848    }
3849
3850    pub async fn stop(&self) -> Result<(), SuperviseError> {
3851        self.stop_requested(None).await
3852    }
3853
3854    pub(crate) async fn stop_requested(
3855        &self,
3856        requester: Option<Principal>,
3857    ) -> Result<(), SuperviseError> {
3858        match self.state()? {
3859            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3860            ModuleState::Starting
3861            | ModuleState::Running
3862            | ModuleState::Unresponsive
3863            | ModuleState::Restarting
3864            | ModuleState::Draining
3865            | ModuleState::Disabled => {}
3866        }
3867
3868        let (reply_tx, reply_rx) = oneshot::channel();
3869        self.inner
3870            .commands
3871            .send(SupervisorCommand::Drain {
3872                requester,
3873                reply: reply_tx,
3874            })
3875            .await
3876            .map_err(|_| SuperviseError::CommandClosed {
3877                module_id: self.inner.module_id.clone(),
3878            })?;
3879        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3880            module_id: self.inner.module_id.clone(),
3881        })?
3882    }
3883
3884    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3885        self.restart_requested(drain_timeout_ms, None).await
3886    }
3887
3888    pub(crate) async fn restart_requested(
3889        &self,
3890        drain_timeout_ms: Option<u64>,
3891        requester: Option<Principal>,
3892    ) -> Result<(), SuperviseError> {
3893        let queued_at = Instant::now();
3894        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3895        let (reply_tx, reply_rx) = oneshot::channel();
3896        self.inner
3897            .commands
3898            .send(SupervisorCommand::Restart {
3899                requester,
3900                drain_timeout_ms,
3901                received_at_generation,
3902                queued_at,
3903                reply: reply_tx,
3904            })
3905            .await
3906            .map_err(|_| SuperviseError::CommandClosed {
3907                module_id: self.inner.module_id.clone(),
3908            })?;
3909        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3910            module_id: self.inner.module_id.clone(),
3911        })?
3912    }
3913
3914    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3915    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3916    /// process then drains in the background of the supervise loop) or has
3917    /// failed, leaving the old process serving.
3918    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3919        self.swap_requested(ready_timeout, None).await
3920    }
3921
3922    pub(crate) async fn swap_requested(
3923        &self,
3924        ready_timeout: Option<Duration>,
3925        requester: Option<Principal>,
3926    ) -> Result<(), SuperviseError> {
3927        let (reply_tx, reply_rx) = oneshot::channel();
3928        self.inner
3929            .commands
3930            .send(SupervisorCommand::Swap {
3931                requester,
3932                ready_timeout,
3933                reply: reply_tx,
3934            })
3935            .await
3936            .map_err(|_| SuperviseError::CommandClosed {
3937                module_id: self.inner.module_id.clone(),
3938            })?;
3939        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3940            module_id: self.inner.module_id.clone(),
3941        })?
3942    }
3943
3944    pub async fn reload(&self) -> Result<(), SuperviseError> {
3945        self.reload_requested(None).await
3946    }
3947
3948    pub(crate) async fn reload_requested(
3949        &self,
3950        requester: Option<Principal>,
3951    ) -> Result<(), SuperviseError> {
3952        let (reply_tx, reply_rx) = oneshot::channel();
3953        self.inner
3954            .commands
3955            .send(SupervisorCommand::Reload {
3956                requester,
3957                reply: reply_tx,
3958            })
3959            .await
3960            .map_err(|_| SuperviseError::CommandClosed {
3961                module_id: self.inner.module_id.clone(),
3962            })?;
3963        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3964            module_id: self.inner.module_id.clone(),
3965        })?
3966    }
3967
3968    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3969        self.set_enabled_requested(enabled, None).await
3970    }
3971
3972    pub(crate) async fn set_enabled_requested(
3973        &self,
3974        enabled: bool,
3975        requester: Option<Principal>,
3976    ) -> Result<bool, SuperviseError> {
3977        let (reply_tx, reply_rx) = oneshot::channel();
3978        self.inner
3979            .commands
3980            .send(SupervisorCommand::SetEnabled {
3981                requester,
3982                enabled,
3983                reply: reply_tx,
3984            })
3985            .await
3986            .map_err(|_| SuperviseError::CommandClosed {
3987                module_id: self.inner.module_id.clone(),
3988            })?;
3989        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3990            module_id: self.inner.module_id.clone(),
3991        })?
3992    }
3993
3994    /// The current process's protocol, or the configured protocol when down.
3995    /// A rescan stores the next launch spec without changing how an existing
3996    /// process registers, serves routes, is probed, or exits.
3997    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3998        let configured = self
3999            .inner
4000            .configuration
4001            .lock()
4002            .map_err(|_| SuperviseError::StatePoisoned {
4003                module_id: Some(self.inner.module_id.clone()),
4004            })?
4005            .spec
4006            .protocol;
4007        let state = lock_snapshot(&self.inner.snapshot)?;
4008        Ok(state.spawned_protocol.unwrap_or(configured))
4009    }
4010
4011    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4012        let configuration =
4013            self.inner
4014                .configuration
4015                .lock()
4016                .map_err(|_| SuperviseError::StatePoisoned {
4017                    module_id: Some(self.inner.module_id.clone()),
4018                })?;
4019        Ok((configuration.spec.clone(), configuration.health.clone()))
4020    }
4021
4022    /// Replace this module's launch spec, keeping its health and drain policy,
4023    /// the way a rescan does for a changed config entry. The running process is
4024    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4025    #[cfg(any(test, feature = "test-support"))]
4026    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4027        let (_, health) = self.configuration()?;
4028        let drain_timeout_ms = u64::try_from(
4029            self.inner
4030                .effective_drain_timeout
4031                .lock()
4032                .unwrap_or_else(|poisoned| poisoned.into_inner())
4033                .as_millis(),
4034        )
4035        .ok();
4036        self.update_configuration(spec, health, drain_timeout_ms)
4037            .await
4038    }
4039
4040    pub(crate) async fn update_configuration(
4041        &self,
4042        spec: ModuleSpec,
4043        health: HealthConfig,
4044        drain_timeout_ms: Option<u64>,
4045    ) -> Result<(), SuperviseError> {
4046        if spec.module_id != self.inner.module_id {
4047            return Err(SuperviseError::InvalidSpec {
4048                reason: "a supervised module's module_id cannot be changed".to_string(),
4049            });
4050        }
4051        validate_spec(&spec)?;
4052        let (reply_tx, reply_rx) = oneshot::channel();
4053        self.inner
4054            .commands
4055            .send(SupervisorCommand::UpdateConfiguration {
4056                spec: spec.clone(),
4057                health: health.clone(),
4058                drain_timeout_ms,
4059                reply: reply_tx,
4060            })
4061            .await
4062            .map_err(|_| SuperviseError::CommandClosed {
4063                module_id: self.inner.module_id.clone(),
4064            })?;
4065        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4066            module_id: self.inner.module_id.clone(),
4067        })?;
4068        let mut configuration =
4069            self.inner
4070                .configuration
4071                .lock()
4072                .map_err(|_| SuperviseError::StatePoisoned {
4073                    module_id: Some(self.inner.module_id.clone()),
4074                })?;
4075        configuration.spec = spec;
4076        configuration.health = health;
4077        Ok(())
4078    }
4079}
4080
4081impl Drop for SupervisedModuleInner {
4082    fn drop(&mut self) {
4083        let Ok(mut monitor) = self.monitor.lock() else {
4084            return;
4085        };
4086        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4087            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4088                state.state = ModuleState::Stopped;
4089                clear_current_process_facts(state);
4090            });
4091            monitor.abort();
4092        }
4093        let _ = monitor.take();
4094    }
4095}
4096
4097#[derive(Debug)]
4098enum SupervisorCommand {
4099    Drain {
4100        requester: Option<Principal>,
4101        reply: oneshot::Sender<Result<(), SuperviseError>>,
4102    },
4103    Retire {
4104        requester: Option<Principal>,
4105        reply: oneshot::Sender<Result<(), SuperviseError>>,
4106    },
4107    Restart {
4108        requester: Option<Principal>,
4109        /// Operator override for this one restart's drain budget, in ms. `None`
4110        /// uses the module's configured/default budget; `Some(0)` cuts
4111        /// immediately (wedge bounce: a stuck request never settles, so
4112        /// waiting only delays recovery).
4113        drain_timeout_ms: Option<u64>,
4114        /// The module's `spawn_generation` when the request was received, before
4115        /// it waited in the command queue. A queued restart whose module has
4116        /// since spawned a newer process is already satisfied (see the handler).
4117        received_at_generation: u64,
4118        /// When the request entered the command queue, so the handler can log
4119        /// how long it waited behind the loop's other work.
4120        queued_at: Instant,
4121        reply: oneshot::Sender<Result<(), SuperviseError>>,
4122    },
4123    Reload {
4124        requester: Option<Principal>,
4125        reply: oneshot::Sender<Result<(), SuperviseError>>,
4126    },
4127    SetEnabled {
4128        requester: Option<Principal>,
4129        enabled: bool,
4130        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4131    },
4132    UpdateConfiguration {
4133        spec: ModuleSpec,
4134        health: HealthConfig,
4135        /// Per-module drain override from the new config; `None` re-resolves to
4136        /// the supervisor-wide default.
4137        drain_timeout_ms: Option<u64>,
4138        reply: oneshot::Sender<()>,
4139    },
4140    Swap {
4141        requester: Option<Principal>,
4142        /// How long the candidate may take to register and declare itself
4143        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4144        ready_timeout: Option<Duration>,
4145        /// Answered at cutover or failure; the incumbent's drain follows.
4146        reply: oneshot::Sender<Result<(), SuperviseError>>,
4147    },
4148}
4149
4150#[derive(Debug)]
4151pub enum SuperviseError {
4152    InvalidSpec {
4153        reason: String,
4154    },
4155    Spawn {
4156        program: PathBuf,
4157        source: io::Error,
4158        cgroup_path: Option<PathBuf>,
4159    },
4160    Cgroup {
4161        module_id: String,
4162        source: io::Error,
4163    },
4164    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4165    /// than spawn a reserved module without its identity binding.
4166    LaunchNonce {
4167        reason: String,
4168    },
4169    Wait {
4170        module_id: String,
4171        source: io::Error,
4172    },
4173    Kill {
4174        module_id: String,
4175        source: io::Error,
4176    },
4177    Forwarding(ForwardingError),
4178    Registry(RegistryError),
4179    ReloadUnavailable {
4180        module_id: String,
4181        reason: String,
4182    },
4183    /// An operator restart/reload was requested for a module that is currently
4184    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4185    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4186    /// by a restart, so these commands are rejected instead of re-enabling it.
4187    Disabled {
4188        module_id: String,
4189    },
4190    ReloadFailed {
4191        module_id: String,
4192        reason: String,
4193    },
4194    RegistrationStillActive {
4195        module_id: String,
4196        waited: Duration,
4197    },
4198    StatePoisoned {
4199        module_id: Option<String>,
4200    },
4201    CommandClosed {
4202        module_id: String,
4203    },
4204    /// A restart or reload arrived while a swap's candidate was warming. The
4205    /// swap owns the module until it cuts over or fails; a stop or disable
4206    /// would have aborted it instead.
4207    SwapInProgress {
4208        module_id: String,
4209    },
4210    /// A swap was refused before anything was spawned.
4211    SwapRefused {
4212        module_id: String,
4213        reason: SwapRefusal,
4214    },
4215    /// A swap spawned a candidate and gave up on it. The candidate has been
4216    /// killed and its slot freed; the incumbent was left serving and was never
4217    /// drained, except in the one `CutoverLost` case described on that arm.
4218    SwapFailed {
4219        module_id: String,
4220        arm: SwapFailureArm,
4221        detail: String,
4222        /// How the candidate exited, when it exited on its own before the
4223        /// supervisor gave up on it.
4224        candidate_exit: Option<ExitReport>,
4225    },
4226}
4227
4228/// Why a swap was refused before a candidate was spawned.
4229#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4230pub enum SwapRefusal {
4231    /// The module's config does not declare `overlap: "safe"`.
4232    OverlapExclusive,
4233    /// The module is not registered, so there is no incumbent to keep serving
4234    /// and nothing a swap would improve on; a plain restart is the tool.
4235    NotRegistered,
4236    /// The module does not speak the subc wire, so a candidate could never
4237    /// register or declare itself ready.
4238    ProtocolNone,
4239    /// The supervisor lacks the forwarding table (to cut routes over) or the
4240    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4241    NotConfigured,
4242    /// A swap is already open for this module.
4243    AlreadySwapping,
4244}
4245
4246impl SwapRefusal {
4247    pub fn as_str(self) -> &'static str {
4248        match self {
4249            Self::OverlapExclusive => "overlap_exclusive",
4250            Self::NotRegistered => "not_registered",
4251            Self::ProtocolNone => "protocol_none",
4252            Self::NotConfigured => "not_configured",
4253            Self::AlreadySwapping => "already_swapping",
4254        }
4255    }
4256}
4257
4258/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4259/// serving and undrained; see `CutoverLost`.
4260#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4261pub enum SwapFailureArm {
4262    /// The candidate process could not be started.
4263    SpawnFailed,
4264    /// The candidate did not register within the readiness budget.
4265    NeverRegistered,
4266    /// The candidate registered but did not declare itself ready in time.
4267    NeverReady,
4268    /// The candidate exited before cutover.
4269    CandidateExited,
4270    /// The candidate declared itself ready but failed its health probe.
4271    CandidateUnhealthy,
4272    /// An operator stop, disable or retire arrived while the candidate warmed.
4273    /// The candidate was killed and the operator's command then carried out on
4274    /// the incumbent.
4275    Interrupted,
4276    /// The candidate's connection closed at the moment of cutover. If it
4277    /// closed before forwarding moved, the incumbent is untouched. If it closed
4278    /// between the forwarding and registry halves of cutover, forwarding can no
4279    /// longer route to the incumbent, so the module is restarted plainly.
4280    CutoverLost,
4281}
4282
4283impl SwapFailureArm {
4284    pub fn as_str(self) -> &'static str {
4285        match self {
4286            Self::SpawnFailed => "spawn_failed",
4287            Self::NeverRegistered => "never_registered",
4288            Self::NeverReady => "never_ready",
4289            Self::CandidateExited => "candidate_exited",
4290            Self::CandidateUnhealthy => "candidate_unhealthy",
4291            Self::Interrupted => "interrupted",
4292            Self::CutoverLost => "cutover_lost",
4293        }
4294    }
4295}
4296
4297impl fmt::Display for SuperviseError {
4298    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4299        match self {
4300            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4301            Self::Spawn {
4302                program,
4303                source,
4304                cgroup_path: Some(cgroup_path),
4305            } => write!(
4306                f,
4307                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4308                cgroup_path.display(),
4309                program.display()
4310            ),
4311            Self::Spawn {
4312                program,
4313                source,
4314                cgroup_path: None,
4315            } => write!(
4316                f,
4317                "failed to spawn module '{}': {source}",
4318                program.display()
4319            ),
4320            Self::Cgroup { module_id, source } => {
4321                write!(
4322                    f,
4323                    "failed to prepare cgroup for module '{module_id}': {source}"
4324                )
4325            }
4326            Self::LaunchNonce { reason } => {
4327                write!(
4328                    f,
4329                    "failed to generate reserved-module launch nonce: {reason}"
4330                )
4331            }
4332            Self::Wait { module_id, source } => {
4333                write!(f, "failed to wait for module '{module_id}': {source}")
4334            }
4335            Self::Kill { module_id, source } => {
4336                write!(f, "failed to kill module '{module_id}': {source}")
4337            }
4338            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4339            Self::Registry(err) => write!(f, "registry error: {err}"),
4340            Self::ReloadUnavailable { module_id, reason } => {
4341                write!(f, "reload unavailable for module '{module_id}': {reason}")
4342            }
4343            Self::Disabled { module_id } => {
4344                write!(
4345                    f,
4346                    "module '{module_id}' is disabled; enable it before restart or reload"
4347                )
4348            }
4349            Self::ReloadFailed { module_id, reason } => {
4350                write!(f, "reload failed for module '{module_id}': {reason}")
4351            }
4352            Self::RegistrationStillActive { module_id, waited } => write!(
4353                f,
4354                "module '{module_id}' registration remained active after waiting {waited:?}"
4355            ),
4356            Self::StatePoisoned { module_id } => match module_id {
4357                Some(module_id) => {
4358                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4359                }
4360                None => write!(f, "supervisor state was poisoned"),
4361            },
4362            Self::CommandClosed { module_id } => {
4363                write!(
4364                    f,
4365                    "supervisor command channel for module '{module_id}' is closed"
4366                )
4367            }
4368            Self::SwapInProgress { module_id } => write!(
4369                f,
4370                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4371            ),
4372            Self::SwapRefused { module_id, reason } => match reason {
4373                SwapRefusal::OverlapExclusive => write!(
4374                    f,
4375                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4376                ),
4377                SwapRefusal::NotRegistered => write!(
4378                    f,
4379                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4380                ),
4381                SwapRefusal::ProtocolNone => write!(
4382                    f,
4383                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4384                ),
4385                SwapRefusal::NotConfigured => write!(
4386                    f,
4387                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4388                ),
4389                SwapRefusal::AlreadySwapping => {
4390                    write!(f, "module '{module_id}' is already being swapped")
4391                }
4392            },
4393            Self::SwapFailed {
4394                module_id,
4395                arm,
4396                detail,
4397                ..
4398            } => write!(
4399                f,
4400                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4401                arm.as_str()
4402            ),
4403        }
4404    }
4405}
4406
4407impl Error for SuperviseError {
4408    fn source(&self) -> Option<&(dyn Error + 'static)> {
4409        match self {
4410            Self::Spawn { source, .. }
4411            | Self::Cgroup { source, .. }
4412            | Self::Wait { source, .. }
4413            | Self::Kill { source, .. } => Some(source),
4414            Self::Forwarding(err) => Some(err),
4415            Self::Registry(err) => Some(err),
4416            Self::LaunchNonce { .. }
4417            | Self::InvalidSpec { .. }
4418            | Self::ReloadUnavailable { .. }
4419            | Self::Disabled { .. }
4420            | Self::ReloadFailed { .. }
4421            | Self::RegistrationStillActive { .. }
4422            | Self::StatePoisoned { .. }
4423            | Self::CommandClosed { .. }
4424            | Self::SwapInProgress { .. }
4425            | Self::SwapRefused { .. }
4426            | Self::SwapFailed { .. } => None,
4427        }
4428    }
4429}
4430
4431pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4432    if spec.module_id.trim().is_empty() {
4433        return Err(SuperviseError::InvalidSpec {
4434            reason: "module_id must not be empty".to_string(),
4435        });
4436    }
4437
4438    Ok(())
4439}
4440
4441#[derive(Debug, Default)]
4442struct HealthProbeRuntime {
4443    configured_health: Option<HealthConfig>,
4444    registered_connection: Option<crate::ConnectionId>,
4445    advertised: bool,
4446    next_probe_at: Option<Instant>,
4447    probe_index: u64,
4448}
4449
4450fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4451    lock_snapshot(snapshot)
4452        .ok()
4453        .and_then(|state| state.spawned_protocol)
4454        .unwrap_or(spec.protocol)
4455}
4456
4457impl HealthProbeRuntime {
4458    fn refresh_registration(
4459        &mut self,
4460        spec: &ModuleSpec,
4461        runtime: &SupervisorRuntimeConfig,
4462        registry: &Registry,
4463        snapshot: &SharedSnapshot,
4464    ) {
4465        if self.configured_health.as_ref() != Some(&runtime.health) {
4466            self.configured_health = Some(runtime.health.clone());
4467            self.next_probe_at = None;
4468            self.registered_connection = None;
4469            self.probe_index = 0;
4470        }
4471        // A non-wire process never registers. Only an explicitly configured
4472        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4473        // health failure for that kind of process.
4474        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4475            self.registered_connection = None;
4476            self.advertised = runtime.health.http.is_some();
4477            if !self.advertised {
4478                self.next_probe_at = None;
4479                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4480                    let unknown = ModuleHealthStatus::default();
4481                    if state.health != unknown {
4482                        state.health = unknown;
4483                    }
4484                });
4485            } else if self.next_probe_at.is_none() {
4486                self.next_probe_at = Some(
4487                    Instant::now()
4488                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4489                );
4490            }
4491            return;
4492        }
4493
4494        let registration = match registry.get_module(&spec.module_id) {
4495            Ok(registration) => registration,
4496            Err(err) => {
4497                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4498                self.advertised = false;
4499                self.next_probe_at = None;
4500                return;
4501            }
4502        };
4503
4504        let Some(registration) = registration else {
4505            self.registered_connection = None;
4506            self.advertised = false;
4507            self.next_probe_at = None;
4508            return;
4509        };
4510
4511        let advertised = registration
4512            .control_ops
4513            .iter()
4514            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4515        if !advertised {
4516            self.registered_connection = Some(registration.connection_id);
4517            self.advertised = false;
4518            self.next_probe_at = None;
4519            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4520                if state.health.status != SupervisorHealthStatus::Unknown
4521                    || state.health.consecutive_failures != 0
4522                    || state.health.last_probe_ms.is_some()
4523                    || state.health.detail.is_some()
4524                    || state.health.metrics.is_some()
4525                {
4526                    state.health.status = SupervisorHealthStatus::Unknown;
4527                    state.health.consecutive_failures = 0;
4528                    state.health.last_probe_ms = None;
4529                    state.health.detail = None;
4530                    state.health.metrics = None;
4531                }
4532            });
4533            return;
4534        }
4535
4536        let reregistered = self.registered_connection != Some(registration.connection_id);
4537        self.registered_connection = Some(registration.connection_id);
4538        self.advertised = true;
4539        if reregistered || self.next_probe_at.is_none() {
4540            self.probe_index = 0;
4541            self.next_probe_at = Some(
4542                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4543            );
4544            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4545                state.health.status = SupervisorHealthStatus::Unknown;
4546                state.health.consecutive_failures = 0;
4547                state.health.detail = None;
4548                state.health.metrics = None;
4549            });
4550        }
4551    }
4552
4553    fn wake_after(&self) -> Option<Duration> {
4554        if !self.advertised {
4555            return None;
4556        }
4557        self.next_probe_at
4558            .map(|next| next.saturating_duration_since(Instant::now()))
4559    }
4560
4561    fn due(&self) -> bool {
4562        self.advertised
4563            && self
4564                .next_probe_at
4565                .is_some_and(|next| Instant::now() >= next)
4566    }
4567
4568    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4569        self.probe_index = self.probe_index.wrapping_add(1);
4570        self.next_probe_at = Some(
4571            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4572        );
4573    }
4574}
4575
4576/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4577///
4578/// This was a struct with a single `message: String`, and every one of the
4579/// fifteen construction sites collapsed into it. Each site knows exactly what it
4580/// saw -- the lane is gone, the module did not answer in time, the module
4581/// answered with the wrong thing -- and `handle_health_probe_failure` then
4582/// treated all of them identically: increment a counter, compare to a threshold,
4583/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4584/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4585///
4586/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4587///
4588/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4589///   answer on it again.
4590/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4591///   AND with a perfectly healthy one that lost a CPU race -- which is what
4592///   happens under machine load, and is how this supervisor killed a healthy
4593///   module three times in one day.
4594/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4595///   Restarting on it is defensible, but it is not the silence case and should
4596///   never be counted as one.
4597/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4598///   anything, so it cannot be evidence about the module at all.
4599///
4600/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4601/// one that fires most often, and while every variant collapsed into one string
4602/// it carried the same weight as the strongest.
4603///
4604/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4605/// DESIGN and a reader stopping at it gets the build backwards: the restart
4606/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4607/// probes still increment the failure streak and drive escalation at the
4608/// threshold (see `is_proof_of_death` below for why that is deliberate and
4609/// what gates the change). Absence of evidence restarts modules today.
4610#[derive(Debug)]
4611enum HealthProbeEvidence {
4612    /// The module's control lane is gone. Proof of death.
4613    LaneDead,
4614    /// No reply within the deadline. Proves nothing about the module's state.
4615    NoAnswer,
4616    /// The module replied, but not with a usable health report. Proves it is alive.
4617    BadAnswer,
4618    /// The daemon could not ask. Says nothing about the module.
4619    Misconfigured,
4620}
4621
4622#[derive(Debug)]
4623struct HealthProbeError {
4624    evidence: HealthProbeEvidence,
4625    message: String,
4626}
4627
4628impl HealthProbeError {
4629    fn lane_dead(message: impl Into<String>) -> Self {
4630        Self::with(HealthProbeEvidence::LaneDead, message)
4631    }
4632
4633    fn no_answer(message: impl Into<String>) -> Self {
4634        Self::with(HealthProbeEvidence::NoAnswer, message)
4635    }
4636
4637    fn bad_answer(message: impl Into<String>) -> Self {
4638        Self::with(HealthProbeEvidence::BadAnswer, message)
4639    }
4640
4641    fn misconfigured(message: impl Into<String>) -> Self {
4642        Self::with(HealthProbeEvidence::Misconfigured, message)
4643    }
4644
4645    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4646        Self {
4647            evidence,
4648            message: message.into(),
4649        }
4650    }
4651
4652    /// Whether this observation is proof the module cannot serve.
4653    ///
4654    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4655    /// variant that fires under CPU starvation, and treating it as proof is the
4656    /// defect this enum exists to make impossible to reintroduce silently.
4657    ///
4658    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4659    /// to restart also needs a bound for the case it excludes -- a genuinely
4660    /// wedged module, alive but never answering -- and that bound must come from
4661    /// the distribution of real late-answer latencies, which nothing measures
4662    /// yet. Landing the classification first makes the later change a one-line
4663    /// decision against evidence that already exists, rather than two unproven
4664    /// changes at once.
4665    #[allow(dead_code)]
4666    fn is_proof_of_death(&self) -> bool {
4667        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4668    }
4669
4670    /// Short stable label for logs and the health snapshot.
4671    ///
4672    /// An operator reading `ck health` currently cannot tell "the module is gone"
4673    /// from "the module did not answer in five seconds", because both render as
4674    /// prose in the same field. These labels are what make the two
4675    /// distinguishable at a glance, and they are what a later restart-policy
4676    /// change will be argued from.
4677    fn label(&self) -> &'static str {
4678        match self.evidence {
4679            HealthProbeEvidence::LaneDead => "lane-dead",
4680            HealthProbeEvidence::NoAnswer => "no-answer",
4681            HealthProbeEvidence::BadAnswer => "bad-answer",
4682            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4683        }
4684    }
4685}
4686
4687impl fmt::Display for HealthProbeError {
4688    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4689        f.write_str(&self.message)
4690    }
4691}
4692
4693async fn run_health_probe_cycle(
4694    spec: &ModuleSpec,
4695    runtime: &SupervisorRuntimeConfig,
4696    registry: &Registry,
4697    process_liveness: &SupervisorProcessLiveness,
4698    snapshot: &SharedSnapshot,
4699    child: &mut Option<SupervisedChild>,
4700) {
4701    let now_ms = unix_ms_now();
4702    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4703        .then_some(runtime.health.http.as_deref())
4704        .flatten();
4705    let result = match http {
4706        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4707        None => probe_module_health(&spec.module_id, runtime, None).await,
4708    };
4709    match result {
4710        Ok(report) => {
4711            handle_health_report(
4712                spec,
4713                runtime,
4714                registry,
4715                process_liveness,
4716                snapshot,
4717                child,
4718                report,
4719                now_ms,
4720            )
4721            .await;
4722        }
4723        Err(err) => {
4724            if http.is_some() {
4725                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4726                    state.health.status = SupervisorHealthStatus::Failing;
4727                });
4728            }
4729            handle_health_probe_failure(
4730                spec,
4731                runtime,
4732                registry,
4733                process_liveness,
4734                snapshot,
4735                child,
4736                err,
4737                now_ms,
4738            )
4739            .await;
4740        }
4741    }
4742}
4743
4744pub(crate) struct HttpProbeTarget<'a> {
4745    address: std::net::SocketAddr,
4746    localhost: bool,
4747    authority: &'a str,
4748    path: String,
4749}
4750
4751/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4752/// or TLS. A URL cannot turn a local health check into an outbound connection.
4753pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4754    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4755        return Err("must not contain whitespace, controls, or a fragment".into());
4756    }
4757    let rest = url
4758        .strip_prefix("http://")
4759        .ok_or("must use plain http://")?;
4760    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4761    let (authority, suffix) = rest.split_at(split);
4762    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4763        ("::1", rest)
4764    } else {
4765        let split = authority.find(':').unwrap_or(authority.len());
4766        authority.split_at(split)
4767    };
4768    let ip: std::net::IpAddr = match host {
4769        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4770        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4771        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4772    };
4773    let port = if port.is_empty() {
4774        80
4775    } else {
4776        port.strip_prefix(':')
4777            .and_then(|p| p.parse::<u16>().ok())
4778            .filter(|p| *p > 0)
4779            .ok_or("must have a valid nonzero TCP port")?
4780    };
4781    let path = if suffix.is_empty() {
4782        "/".into()
4783    } else if suffix.starts_with('?') {
4784        format!("/{suffix}")
4785    } else {
4786        suffix.into()
4787    };
4788    Ok(HttpProbeTarget {
4789        address: std::net::SocketAddr::new(ip, port),
4790        localhost: host == "localhost",
4791        authority,
4792        path,
4793    })
4794}
4795
4796async fn probe_http_health(
4797    url: &str,
4798    deadline: Duration,
4799) -> Result<HealthReport, HealthProbeError> {
4800    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4801    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4802    // Keep partial diagnostics outside the timed future so cancellation does
4803    // not discard a status line or body bytes already received.
4804    let mut response_status = String::new();
4805    let mut body = Vec::new();
4806    let probe = async {
4807        // Resolve localhost ourselves so a hosts-file override cannot turn
4808        // this into an outbound request, while IPv6-only local servers work.
4809        let connection = match tokio::net::TcpStream::connect(target.address).await {
4810            Err(_) if target.localhost => {
4811                tokio::net::TcpStream::connect((
4812                    std::net::Ipv6Addr::LOCALHOST,
4813                    target.address.port(),
4814                ))
4815                .await
4816            }
4817            result => result,
4818        };
4819        let mut stream = connection.map_err(|error| {
4820            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4821        })?;
4822        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4823            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4824        let mut reader = BufReader::new(stream);
4825        let mut budget = 16 * 1024;
4826        let status = http_line(&mut reader, &mut budget).await?;
4827        let mut words = status.split_ascii_whitespace();
4828        let version = words.next();
4829        let code = words
4830            .next()
4831            .filter(|word| word.len() == 3)
4832            .and_then(|word| word.parse::<u16>().ok());
4833        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4834            || !code.is_some_and(|code| (100..600).contains(&code))
4835        {
4836            return Err(HealthProbeError::bad_answer(format!(
4837                "invalid HTTP status: {status}"
4838            )));
4839        }
4840        let code = code.expect("validated status code");
4841        response_status = status.clone();
4842        let mut length = None;
4843        let mut chunked = false;
4844        loop {
4845            let line = http_line(&mut reader, &mut budget).await?;
4846            if line.is_empty() {
4847                break;
4848            }
4849            if let Some((name, value)) = line.split_once(':') {
4850                if name.eq_ignore_ascii_case("content-length") {
4851                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4852                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4853                    })?);
4854                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4855                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4856                }
4857            }
4858        }
4859        if chunked {
4860            while body.len() < 200 {
4861                let line = http_line(&mut reader, &mut budget).await?;
4862                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4863                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4864                if size == 0 {
4865                    break;
4866                }
4867                let count = size.min((200 - body.len()) as u64) as usize;
4868                let start = body.len();
4869                (&mut reader)
4870                    .take(count as u64)
4871                    .read_to_end(&mut body)
4872                    .await
4873                    .map_err(|error| {
4874                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4875                    })?;
4876                if body.len() - start != count {
4877                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4878                }
4879                if size > count as u64 || body.len() == 200 {
4880                    break;
4881                }
4882                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4883                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4884                }
4885            }
4886        } else {
4887            reader
4888                .take(length.unwrap_or(200).min(200))
4889                .read_to_end(&mut body)
4890                .await
4891                .map_err(|error| {
4892                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4893                })?;
4894        }
4895        if (200..300).contains(&code) {
4896            Ok(HealthReport::ok())
4897        } else {
4898            Err(HealthProbeError::bad_answer(
4899                "HTTP health endpoint returned non-2xx",
4900            ))
4901        }
4902    };
4903    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4904        Err(HealthProbeError::no_answer(format!(
4905            "HTTP probe timed out after {deadline:?}"
4906        )))
4907    });
4908    if let Err(error) = &mut result {
4909        if !response_status.is_empty() {
4910            error.message = format!(
4911                "{}; {response_status}: {}",
4912                error.message,
4913                String::from_utf8_lossy(&body)
4914            );
4915        }
4916    }
4917    result
4918}
4919
4920async fn http_line(
4921    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4922    remaining: &mut usize,
4923) -> Result<String, HealthProbeError> {
4924    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4925    let mut line = Vec::new();
4926    (&mut *reader)
4927        .take(*remaining as u64)
4928        .read_until(b'\n', &mut line)
4929        .await
4930        .map_err(|error| {
4931            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4932        })?;
4933    *remaining -= line.len();
4934    if !line.ends_with(b"\r\n") {
4935        return Err(HealthProbeError::bad_answer(
4936            "HTTP headers are incomplete or exceed 16 KiB",
4937        ));
4938    }
4939    line.truncate(line.len() - 2);
4940    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4941}
4942
4943async fn probe_module_health(
4944    module_id: &str,
4945    runtime: &SupervisorRuntimeConfig,
4946    drain_deadline: Option<Instant>,
4947) -> Result<HealthReport, HealthProbeError> {
4948    let Some(forwarding) = runtime.forwarding.as_ref() else {
4949        return Err(HealthProbeError::misconfigured(
4950            "supervisor was not configured with a forwarding table",
4951        ));
4952    };
4953    let probe_started_at = Instant::now();
4954    let mut deadline = probe_started_at + runtime.health.deadline;
4955    if let Some(drain_deadline) = drain_deadline {
4956        deadline = deadline.min(drain_deadline);
4957    }
4958    let pending = if drain_deadline.is_some() {
4959        forwarding.begin_drain_health_probe_rpc_for(
4960            module_id,
4961            MODULE_CONTROL_OP_HEALTH_CHECK,
4962            probe_started_at,
4963            deadline,
4964        )
4965    } else {
4966        forwarding.begin_health_probe_rpc_for(
4967            module_id,
4968            MODULE_CONTROL_OP_HEALTH_CHECK,
4969            probe_started_at,
4970            deadline,
4971        )
4972    }
4973    .map_err(|err| {
4974        // The endpoint is not registered, so there is no live control lane to
4975        // ask. That is the module being absent, not slow.
4976        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4977    })?;
4978    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4979}
4980
4981/// [`probe_module_health`] for one endpoint rather than the id's active one.
4982///
4983/// A swap probes two processes that no by-id lookup reaches: its candidate
4984/// before cutover, and its superseded incumbent (for busy gauges) while the
4985/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
4986/// bounds the by-id drain probe.
4987async fn probe_endpoint_health(
4988    endpoint: crate::ModuleEndpointId,
4989    runtime: &SupervisorRuntimeConfig,
4990    deadline_cap: Option<Instant>,
4991) -> Result<HealthReport, HealthProbeError> {
4992    let Some(forwarding) = runtime.forwarding.as_ref() else {
4993        return Err(HealthProbeError::misconfigured(
4994            "supervisor was not configured with a forwarding table",
4995        ));
4996    };
4997    let probe_started_at = Instant::now();
4998    let mut deadline = probe_started_at + runtime.health.deadline;
4999    if let Some(cap) = deadline_cap {
5000        deadline = deadline.min(cap);
5001    }
5002    let pending = forwarding
5003        .begin_endpoint_health_probe_rpc_for(
5004            endpoint,
5005            MODULE_CONTROL_OP_HEALTH_CHECK,
5006            probe_started_at,
5007            deadline,
5008        )
5009        .map_err(|err| {
5010            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5011        })?;
5012    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5013}
5014
5015/// Send a begun health probe and classify its answer.
5016async fn await_health_probe(
5017    forwarding: &ForwardingTable,
5018    pending: PendingModuleControlRpc,
5019    deadline: Instant,
5020    probe_budget: Duration,
5021) -> Result<HealthReport, HealthProbeError> {
5022    let PendingModuleControlRpc {
5023        endpoint,
5024        module_sink,
5025        negotiated_ver,
5026        corr,
5027        receiver,
5028    } = pending;
5029    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5030        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5031    })?;
5032    let frame = Frame::build_with_version(
5033        negotiated_ver,
5034        FrameType::Request,
5035        control_flags(),
5036        0,
5037        0,
5038        corr,
5039        body,
5040    )
5041    .map_err(|err| {
5042        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5043    })?;
5044
5045    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5046    // blocks waiting for capacity when the module's egress queue is full, and an
5047    // unbounded await here freezes the whole supervision actor (it stops polling
5048    // Child::wait and supervisor commands), making the module unrecoverable
5049    // in-band. On timeout the probe fails like any transport failure.
5050    match timeout_at(deadline, module_sink.send(frame)).await {
5051        Ok(Ok(())) => {}
5052        Ok(Err(err)) => {
5053            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5054            // A closed sink means the module's egress channel is gone -- the
5055            // receiving half is dropped when its connection tears down. Proof.
5056            return Err(HealthProbeError::lane_dead(format!(
5057                "failed to send health.check: {err}"
5058            )));
5059        }
5060        Err(_elapsed) => {
5061            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5062            // A full egress queue means the module is not draining its socket, which
5063            // is consistent with a wedged module AND with one whose reader is merely
5064            // starved. Silence, not proof.
5065            return Err(HealthProbeError::no_answer(
5066                "health.check send timed out before enqueue (module egress full)",
5067            ));
5068        }
5069    }
5070
5071    match timeout_at(deadline, receiver).await {
5072        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5073        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5074        // and those prove it is alive even though the probe failed.
5075        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5076            response.health_report().ok_or_else(|| {
5077                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5078            })
5079        }
5080        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5081            format!("health.check rejected: {}", body.message),
5082        )),
5083        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5084            Err(HealthProbeError::lane_dead(message))
5085        }
5086        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5087            Err(HealthProbeError::bad_answer(message))
5088        }
5089        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5090            Err(HealthProbeError::bad_answer(format!(
5091                "expected module-control op '{expected}', got '{actual}'"
5092            )))
5093        }
5094        // A reply that crosses the deadline before this waiter observes it is
5095        // still proof of life. The forwarding path records its end-to-end latency
5096        // before delivering this classification.
5097        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5098            "module answered health.check after its daemon deadline",
5099        )),
5100        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5101            "health.check waiter was canceled before the module responded",
5102        )),
5103        Err(_) => {
5104            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5105            Err(HealthProbeError::no_answer(format!(
5106                "module did not answer health.check within {probe_budget:?}"
5107            )))
5108        }
5109    }
5110}
5111
5112#[allow(clippy::too_many_arguments)]
5113async fn handle_health_report(
5114    spec: &ModuleSpec,
5115    runtime: &SupervisorRuntimeConfig,
5116    registry: &Registry,
5117    process_liveness: &SupervisorProcessLiveness,
5118    snapshot: &SharedSnapshot,
5119    child: &mut Option<SupervisedChild>,
5120    report: HealthReport,
5121    now_ms: u64,
5122) {
5123    let status = supervisor_health_status(report.status);
5124    let detail = report.detail.clone();
5125    let metrics = truncate_health_metrics(report.metrics);
5126    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5127        state.health.status = status;
5128        state.health.last_probe_ms = Some(now_ms);
5129        state.health.detail = detail.clone();
5130        state.health.metrics = metrics.clone();
5131        state.health.consecutive_failures = 0;
5132    });
5133
5134    let action = match report.status {
5135        HealthStatus::Ok => return,
5136        HealthStatus::Degraded => runtime.health.on_degraded,
5137        HealthStatus::Failing => runtime.health.on_failing,
5138    };
5139    apply_l3_health_action(
5140        spec,
5141        runtime,
5142        registry,
5143        process_liveness,
5144        snapshot,
5145        child,
5146        status,
5147        detail.as_deref(),
5148        action,
5149        now_ms,
5150    )
5151    .await;
5152}
5153
5154#[allow(clippy::too_many_arguments)]
5155async fn handle_health_probe_failure(
5156    spec: &ModuleSpec,
5157    runtime: &SupervisorRuntimeConfig,
5158    registry: &Registry,
5159    process_liveness: &SupervisorProcessLiveness,
5160    snapshot: &SharedSnapshot,
5161    child: &mut Option<SupervisedChild>,
5162    err: HealthProbeError,
5163    now_ms: u64,
5164) {
5165    let threshold = runtime.health.failure_threshold.max(1);
5166    let mut failures = 0;
5167    // Carry the evidence class into the operator-visible detail. Without it,
5168    // "module did not answer within 5s" and "the control lane is gone" are two
5169    // prose strings in the same field, and the reader has to know the codebase to
5170    // tell which one is proof of anything.
5171    let detail = format!("[{}] {err}", err.label());
5172    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5173        // A failed wire probe invalidates the last report, even before the
5174        // restart threshold. HTTP probes already mark failures as Failing.
5175        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5176            state.health.status = SupervisorHealthStatus::Unknown;
5177        }
5178        state.health.last_probe_ms = Some(now_ms);
5179        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5180        state.health.detail = Some(detail.clone());
5181        state.health.metrics = None;
5182        failures = state.health.consecutive_failures;
5183    });
5184
5185    if failures < threshold {
5186        warn!(
5187            module_id = %spec.module_id,
5188            consecutive_failures = failures,
5189            threshold,
5190            evidence = err.label(),
5191            detail = %detail,
5192            "health.check probe failed"
5193        );
5194        return;
5195    }
5196
5197    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5198        state.state = ModuleState::Unresponsive;
5199        state.health.status = SupervisorHealthStatus::Unresponsive;
5200    });
5201    // The evidence class is logged at the kill site because this is the line an
5202    // operator reads after an unexplained restart. A streak of `no-answer` under
5203    // machine load is the known false-positive shape; a `lane-dead` is not.
5204    if runtime.health.critical {
5205        error!(
5206            module_id = %spec.module_id,
5207            status = "unresponsive",
5208            evidence = err.label(),
5209            detail = %detail,
5210            "critical module health alert"
5211        );
5212    } else {
5213        warn!(
5214            module_id = %spec.module_id,
5215            status = "unresponsive",
5216            evidence = err.label(),
5217            detail = %detail,
5218            "module health threshold breached"
5219        );
5220    }
5221    if let Err(err) = health_restart_child(
5222        spec,
5223        runtime,
5224        registry,
5225        process_liveness,
5226        snapshot,
5227        child,
5228        SupervisorHealthStatus::Unresponsive,
5229        Some(&detail),
5230        now_ms,
5231    )
5232    .await
5233    {
5234        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5235    }
5236}
5237
5238#[allow(clippy::too_many_arguments)]
5239async fn apply_l3_health_action(
5240    spec: &ModuleSpec,
5241    runtime: &SupervisorRuntimeConfig,
5242    registry: &Registry,
5243    process_liveness: &SupervisorProcessLiveness,
5244    snapshot: &SharedSnapshot,
5245    child: &mut Option<SupervisedChild>,
5246    status: SupervisorHealthStatus,
5247    detail: Option<&str>,
5248    action: HealthAction,
5249    now_ms: u64,
5250) {
5251    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5252    match action {
5253        HealthAction::Report => {
5254            info!(
5255                module_id = %spec.module_id,
5256                status = ?status,
5257                detail,
5258                "module reported non-ok health"
5259            );
5260        }
5261        HealthAction::Alert => {
5262            error!(
5263                module_id = %spec.module_id,
5264                status = ?status,
5265                detail,
5266                "module health alert"
5267            );
5268        }
5269        HealthAction::Restart => {
5270            if let Err(err) = health_restart_child(
5271                spec,
5272                runtime,
5273                registry,
5274                process_liveness,
5275                snapshot,
5276                child,
5277                status,
5278                detail,
5279                now_ms,
5280            )
5281            .await
5282            {
5283                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5284            }
5285        }
5286    }
5287}
5288
5289#[allow(clippy::too_many_arguments)]
5290async fn health_restart_child(
5291    spec: &ModuleSpec,
5292    runtime: &SupervisorRuntimeConfig,
5293    registry: &Registry,
5294    process_liveness: &SupervisorProcessLiveness,
5295    snapshot: &SharedSnapshot,
5296    child: &mut Option<SupervisedChild>,
5297    status: SupervisorHealthStatus,
5298    detail: Option<&str>,
5299    now_ms: u64,
5300) -> Result<(), SuperviseError> {
5301    let (enabled, schedule) = {
5302        let mut state = lock_snapshot(snapshot)?;
5303        let enabled = state.enabled;
5304        let schedule = if enabled {
5305            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5306        } else {
5307            None
5308        };
5309        (enabled, schedule)
5310    };
5311
5312    if !enabled {
5313        return Err(SuperviseError::Disabled {
5314            module_id: spec.module_id.clone(),
5315        });
5316    }
5317
5318    if schedule.is_none() {
5319        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5320        error!(
5321            module_id = %spec.module_id,
5322            status = ?status,
5323            detail,
5324            max_restarts = runtime.restart_policy.max_restarts,
5325            window_secs = runtime.restart_policy.window.as_secs(),
5326            reason = %runtime.restart_policy.budget_exhausted_detail(),
5327            "health restart budget exhausted; marking module failed"
5328        );
5329        let stop_notice = begin_forwarding_drain_if_configured(
5330            spec,
5331            runtime,
5332            registry,
5333            snapshot,
5334            Some(true),
5335            RouteCloseReason::Disable,
5336        )
5337        .await?;
5338        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5339            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5340        })?;
5341        drain_optional_child(
5342            &spec.module_id,
5343            spec.protocol,
5344            stop_notice,
5345            registry,
5346            runtime.forwarding.as_deref(),
5347            snapshot,
5348            &runtime.terminal_ring,
5349            &runtime.spawn_events,
5350            child,
5351            runtime.drain_timeout,
5352            ModuleState::Failed,
5353            Some(true),
5354        )
5355        .await?;
5356        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5357        return Ok(());
5358    }
5359
5360    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5361    let mut restart_count = 0;
5362    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5363        restart_count = state.crash_restarts.len();
5364        state.state = ModuleState::Unresponsive;
5365        state.health.status = status;
5366        state.health.last_action = Some(HealthAction::Restart.to_string());
5367        state.health.last_action_ms = Some(now_ms);
5368    })?;
5369    warn!(
5370        module_id = %spec.module_id,
5371        status = ?status,
5372        detail,
5373        restart_count,
5374        restart_in_window = schedule.restart_in_window,
5375        delay_ms = schedule.delay.as_millis() as u64,
5376        "health-triggered module restart"
5377    );
5378
5379    let stop_notice = begin_forwarding_drain_if_configured(
5380        spec,
5381        runtime,
5382        registry,
5383        snapshot,
5384        Some(true),
5385        RouteCloseReason::Restart,
5386    )
5387    .await?;
5388    drain_optional_child(
5389        &spec.module_id,
5390        spec.protocol,
5391        stop_notice,
5392        registry,
5393        runtime.forwarding.as_deref(),
5394        snapshot,
5395        &runtime.terminal_ring,
5396        &runtime.spawn_events,
5397        child,
5398        runtime.drain_timeout,
5399        ModuleState::Restarting,
5400        Some(true),
5401    )
5402    .await?;
5403    schedule_respawn(
5404        runtime,
5405        snapshot,
5406        &spec.module_id,
5407        schedule.delay,
5408        RespawnKind::Spawn,
5409    )
5410}
5411
5412fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5413    if let Some(reply) = runtime
5414        .deferred_reload_reply
5415        .lock()
5416        .unwrap_or_else(|p| p.into_inner())
5417        .take()
5418    {
5419        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5420            module_id: module_id.to_string(),
5421            reason: reason.to_string(),
5422        }));
5423    }
5424}
5425
5426fn schedule_respawn(
5427    runtime: &SupervisorRuntimeConfig,
5428    snapshot: &SharedSnapshot,
5429    module_id: &str,
5430    delay: Duration,
5431    kind: RespawnKind,
5432) -> Result<(), SuperviseError> {
5433    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5434    update_snapshot(snapshot, Some(module_id), |state| {
5435        state.respawn_pending = true
5436    })?;
5437    *runtime
5438        .scheduled_respawn
5439        .lock()
5440        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5441        deadline: Instant::now() + delay,
5442        kind,
5443    });
5444    Ok(())
5445}
5446
5447fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5448    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5449        state.health.last_action = Some(action);
5450        state.health.last_action_ms = Some(now_ms);
5451    });
5452}
5453
5454fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5455    match status {
5456        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5457        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5458        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5459    }
5460}
5461
5462/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5463/// returned to every `supervisor.list` and `supervisor.health` caller.
5464///
5465/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5466/// path: that request exists to return a module's complete metrics object, and
5467/// `ck health <module-id>` documents it as the way to see what the cached view
5468/// truncates. The asymmetry is the feature.
5469///
5470/// So a new caller must decide which side it is on rather than assume the cap is
5471/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5472/// the truncation that path exists to avoid.
5473fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5474    let metrics = metrics?;
5475    match serde_json::to_vec(&metrics) {
5476        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5477            "truncated": true,
5478            "original_bytes": encoded.len(),
5479        })),
5480        Ok(_) | Err(_) => Some(metrics),
5481    }
5482}
5483
5484/// Spread health probes so a fleet-wide restart does not converge them.
5485///
5486/// The delay is derived from the module id and probe index rather than a random
5487/// source, so it is deterministic per module: a module keeps its own offset
5488/// across daemon restarts instead of re-rolling into a collision.
5489fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5490    if cadence.is_zero() {
5491        return Duration::ZERO;
5492    }
5493    let cadence_ms = cadence.as_millis() as u64;
5494    // This early return is REDUNDANT, deliberately, and a mutation run will show
5495    // it surviving removal. Recording why here so the next person to notice does
5496    // not have to re-derive it:
5497    //
5498    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5499    //   a zero cadence and builds the Duration from whole milliseconds, so a
5500    //   sub-millisecond cadence cannot come from config.
5501    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5502    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5503    //   -- exactly what this returns.
5504    //
5505    // Kept as a guard against a future widening of the config parser (accepting
5506    // microseconds, say), which would make the sub-millisecond case reachable.
5507    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5508    // divides by zero. Remove this and nothing changes.
5509    if cadence_ms == 0 {
5510        return cadence;
5511    }
5512    // Note that this never returns less than one cadence, including for the FIRST
5513    // probe. So a freshly registered module reports health `unknown` for a full
5514    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5515    // ready to answer.
5516    //
5517    // That is a property of the supervisor's schedule, not of any module: an
5518    // operator watching a restart sees `unknown` and cannot tell it from a module
5519    // that is slow to warm. Measured on two unrelated modules, both flipping to
5520    // `ok` between 22s and 32s after restart.
5521    //
5522    // Left as-is because spreading the first probe is what keeps a fleet-wide
5523    // restart from firing fourteen simultaneous probes into a cold machine. The
5524    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5525    // that thundering herd for a faster first reading.
5526    let jitter_span = (cadence_ms / 10).max(1);
5527    let hash = module_id.as_bytes().iter().fold(
5528        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5529        |acc, byte| {
5530            acc.wrapping_mul(1099511628211)
5531                .wrapping_add(u64::from(*byte))
5532        },
5533    );
5534    cadence + Duration::from_millis(hash % jitter_span)
5535}
5536
5537#[cfg(test)]
5538mod tests {
5539    use super::*;
5540
5541    #[test]
5542    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5543        let handle = SupervisorHandle::new();
5544        let module_id = "readded-tombstone";
5545        handle.record_rescan_removal(module_id);
5546        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5547
5548        handle.apply_identity_configuration(&ModuleSpec {
5549            module_id: module_id.to_string(),
5550            program: PathBuf::from("/test/module"),
5551            args: Vec::new(),
5552            env: Vec::new(),
5553            reserved: false,
5554            reserved_prefixes: Vec::new(),
5555            protocol: ModuleProtocol::Subc,
5556            overlap: Default::default(),
5557        });
5558
5559        assert!(
5560            handle.removal_tombstone_age_ms(module_id).is_none(),
5561            "a re-added module must not retain a stale removal tombstone"
5562        );
5563    }
5564
5565    /// What one module's owner looked like from the control plane at the
5566    /// instant after its first process was spawned.
5567    #[derive(Debug, PartialEq, Eq)]
5568    struct OwnerInSpawnWindow {
5569        module_id: String,
5570        configured: bool,
5571        on_roster: bool,
5572        admission_refusal: Option<&'static str>,
5573    }
5574
5575    /// A supervised module's process can connect, register, sync its scopes
5576    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5577    /// the module on the roster until afterward; configured supervision does
5578    /// so before spawning. In both cases the owner must
5579    /// already read as configured, so a scoped `route.open` against it is
5580    /// refused as retryable `scope_not_synced` and not as terminal
5581    /// `scope_not_live` ("will never sync").
5582    ///
5583    /// The hook runs just after each initial spawn attempt, before returning to
5584    /// the caller, so no race with a real child's registration is needed.
5585    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5586    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5587        use crate::scopes::ScopeTable;
5588        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5589
5590        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5591        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5592            module_id: module_id.to_string(),
5593            program,
5594            args: Vec::new(),
5595            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5596                .into_iter()
5597                .map(|key| (key.to_string(), dir.path().display().to_string()))
5598                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5599                .collect(),
5600            reserved: false,
5601            reserved_prefixes: Vec::new(),
5602            protocol: ModuleProtocol::Subc,
5603            overlap: Default::default(),
5604        };
5605        let live = super::terminal_history_tests::fake_aft_stub_path();
5606        let missing = dir.path().join("definitely-missing-module");
5607
5608        let handle = SupervisorHandle::new();
5609        let mut supervisor =
5610            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5611                .with_handle(handle.clone());
5612        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5613        let hook_handle = handle.clone();
5614        let hook_observed = Arc::clone(&observed);
5615        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5616            // Exactly what the control plane computes for a scoped route.open
5617            // naming this module as the owner of a scope it has not synced.
5618            let configured = hook_handle.is_configured(module_id);
5619            let selector = ScopeSelector::new(
5620                Principal::Reserved {
5621                    module_id: module_id.to_string(),
5622                },
5623                "s".to_string(),
5624                Some(1),
5625            );
5626            let carrier = Principal::Reserved {
5627                module_id: "carrier".to_string(),
5628            };
5629            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5630                .admit(&carrier, module_id, &selector, configured)
5631            {
5632                Ok(_) => None,
5633                Err(refusal) => Some(refusal.code),
5634            };
5635            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5636                module_id: module_id.to_string(),
5637                configured,
5638                on_roster: hook_handle.get(module_id).is_some(),
5639                admission_refusal,
5640            });
5641        })));
5642
5643        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5644        let configured = supervisor
5645            .supervise_configured(stub("configured", live.clone()), true)
5646            .unwrap();
5647        let with_health = supervisor
5648            .supervise_configured_with_health(
5649                stub("with-health", live.clone()),
5650                true,
5651                HealthConfig::default(),
5652                None,
5653                RestartPolicy::default(),
5654            )
5655            .unwrap();
5656        // The failed-spawn path still puts the module on the roster (as
5657        // failed), so it is configured throughout.
5658        let failed = supervisor
5659            .supervise_configured_with_health(
5660                stub("failed-spawn", missing.clone()),
5661                true,
5662                HealthConfig::default(),
5663                None,
5664                RestartPolicy::default(),
5665            )
5666            .unwrap();
5667        // A failed plain `spawn` puts nothing on the roster, so its mark is
5668        // taken back once the spawn has failed.
5669        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5670
5671        let expected = [
5672            "plain",
5673            "configured",
5674            "with-health",
5675            "failed-spawn",
5676            "spawn-error",
5677        ]
5678        .into_iter()
5679        .map(|module_id| OwnerInSpawnWindow {
5680            module_id: module_id.to_string(),
5681            configured: true,
5682            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5683            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5684        })
5685        .collect::<Vec<_>>();
5686        assert_eq!(*observed.lock().unwrap(), expected);
5687
5688        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5689            assert!(
5690                handle.get(module_id).is_some(),
5691                "{module_id} is on the roster"
5692            );
5693            assert!(
5694                handle.is_configured(module_id),
5695                "{module_id} stays configured"
5696            );
5697        }
5698        assert!(handle.get("spawn-error").is_none());
5699        assert!(
5700            !handle.is_configured("spawn-error"),
5701            "a plain spawn that failed must not leave its module marked configured"
5702        );
5703
5704        // Leaving the roster clears the mark with it.
5705        handle.retire("failed-spawn");
5706        assert!(!handle.is_configured("failed-spawn"));
5707
5708        for module in [plain, configured, with_health] {
5709            module.stop().await.unwrap();
5710        }
5711        drop(failed);
5712    }
5713
5714    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5715        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5716        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5717            snapshot.process_alive = true;
5718            snapshot.pid = Some(41);
5719            snapshot.spawned_at_ms = Some(42);
5720            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5721            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5722            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5723                device: 43,
5724                inode: 44,
5725                #[cfg(windows)]
5726                windows_image: None,
5727            });
5728        })
5729        .unwrap();
5730        snapshot
5731    }
5732
5733    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5734        let snapshot = lock_snapshot(snapshot).unwrap();
5735        assert!(!snapshot.process_alive);
5736        assert_eq!(snapshot.pid, None);
5737        assert_eq!(snapshot.spawned_at_ms, None);
5738        assert_eq!(snapshot.spawned_from, None);
5739        assert_eq!(snapshot.launch_spec_digest, None);
5740        assert_eq!(snapshot.spawned_file_identity, None);
5741    }
5742
5743    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5744    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5745        let supervisor =
5746            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5747        let mut runtime = supervisor.runtime_config();
5748        runtime.test_seed_stale_facts_before_enable_spawn = true;
5749        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5750        let mut child = None;
5751        let spec = ModuleSpec {
5752            module_id: "failed-enable-clears-facts".to_string(),
5753            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5754            args: Vec::new(),
5755            env: Vec::new(),
5756            reserved: false,
5757            reserved_prefixes: Vec::new(),
5758            protocol: ModuleProtocol::Subc,
5759            overlap: Default::default(),
5760        };
5761
5762        let result = set_child_enabled(
5763            &spec,
5764            &runtime,
5765            &supervisor.registry,
5766            &supervisor.process_liveness,
5767            &snapshot,
5768            &mut child,
5769            true,
5770        )
5771        .await;
5772
5773        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5774        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5775        assert_snapshot_process_facts_cleared(&snapshot);
5776    }
5777
5778    #[tokio::test]
5779    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5780        let supervisor =
5781            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5782        let runtime = supervisor.runtime_config();
5783        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5784            ModuleState::Restarting,
5785            true,
5786        )));
5787        let spec = ModuleSpec {
5788            module_id: "start-stranded-restarting".to_string(),
5789            program: super::terminal_history_tests::fake_aft_stub_path(),
5790            args: Vec::new(),
5791            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5792            reserved: false,
5793            reserved_prefixes: Vec::new(),
5794            protocol: ModuleProtocol::None,
5795            overlap: Default::default(),
5796        };
5797        let mut child = None;
5798        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5799        assert!(!super::set_child_enabled(
5800            &spec,
5801            &runtime,
5802            &Registry::default(),
5803            &supervisor.process_liveness,
5804            &snapshot,
5805            &mut child,
5806            true
5807        )
5808        .await
5809        .unwrap());
5810        assert!(child.is_none());
5811        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5812        assert!(super::set_child_enabled(
5813            &spec,
5814            &runtime,
5815            &Registry::default(),
5816            &supervisor.process_liveness,
5817            &snapshot,
5818            &mut child,
5819            true
5820        )
5821        .await
5822        .unwrap());
5823        assert_eq!(
5824            lock_snapshot(&snapshot).unwrap().state,
5825            ModuleState::Running
5826        );
5827        let mut child = child.unwrap();
5828        child.start_kill().unwrap();
5829        child.wait().await.unwrap();
5830    }
5831
5832    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5833    async fn failed_reload_spawn_clears_current_process_facts() {
5834        let supervisor =
5835            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5836        let mut runtime = supervisor.runtime_config();
5837        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5838        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5839        let mut child = None;
5840        let spec = ModuleSpec {
5841            module_id: "failed-reload-clears-facts".to_string(),
5842            program: PathBuf::from("/unused/failed-reload-module"),
5843            args: Vec::new(),
5844            env: Vec::new(),
5845            reserved: false,
5846            reserved_prefixes: Vec::new(),
5847            protocol: ModuleProtocol::Subc,
5848            overlap: Default::default(),
5849        };
5850
5851        let result = handle_reload_spawn_failure(
5852            &spec,
5853            &runtime,
5854            &supervisor.process_liveness,
5855            &snapshot,
5856            &mut child,
5857            "forced reload spawn failure".to_string(),
5858        )
5859        .await;
5860
5861        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5862        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5863        assert_snapshot_process_facts_cleared(&snapshot);
5864    }
5865
5866    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5867    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5868        let supervisor =
5869            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5870        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5871        let module = supervisor.supervised_module(
5872            ModuleSpec {
5873                module_id: "drop-clears-facts".to_string(),
5874                program: PathBuf::from("/unused/drop-module"),
5875                args: Vec::new(),
5876                env: Vec::new(),
5877                reserved: false,
5878                reserved_prefixes: Vec::new(),
5879                protocol: ModuleProtocol::Subc,
5880                overlap: Default::default(),
5881            },
5882            supervisor.runtime_config(),
5883            Arc::clone(&snapshot),
5884            None,
5885        );
5886        assert!(!module
5887            .inner
5888            .monitor
5889            .lock()
5890            .unwrap()
5891            .as_ref()
5892            .unwrap()
5893            .is_finished());
5894
5895        drop(module);
5896
5897        assert_eq!(
5898            lock_snapshot(&snapshot).unwrap().state,
5899            ModuleState::Stopped
5900        );
5901        assert_snapshot_process_facts_cleared(&snapshot);
5902    }
5903
5904    #[cfg(unix)]
5905    #[tokio::test]
5906    async fn rescan_preserves_running_protocol_until_respawn() {
5907        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5908        let initial = ModuleSpec {
5909            module_id: "rescan-protocol".into(),
5910            program: PathBuf::from("/bin/sleep"),
5911            args: vec!["60".into()],
5912            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5913                .into_iter()
5914                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5915                .collect(),
5916            reserved: false,
5917            reserved_prefixes: vec![],
5918            protocol: ModuleProtocol::None,
5919            overlap: Default::default(),
5920        };
5921        let supervisor =
5922            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5923        let module = supervisor.spawn(initial.clone()).unwrap();
5924        assert!(module.status().unwrap().live);
5925        let mut next = initial;
5926        next.protocol = ModuleProtocol::Subc;
5927        module
5928            .update_configuration(next.clone(), HealthConfig::default(), None)
5929            .await
5930            .unwrap();
5931        assert!(
5932            module.status().unwrap().live,
5933            "rescan must not require HELLO from the old non-wire process"
5934        );
5935        let runtime = supervisor.runtime_config();
5936        let action = on_child_exit(
5937            &next,
5938            RestartPolicy::default(),
5939            &supervisor.registry,
5940            &module.inner.snapshot,
5941            &runtime.terminal_ring,
5942            &runtime.spawn_events,
5943            &runtime.child_roster,
5944            ExitReport {
5945                kind: ExitKind::Clean,
5946                code: Some(0),
5947                signal: None,
5948                at_ms: unix_ms_now(),
5949            },
5950        )
5951        .await;
5952        assert!(
5953            matches!(action, NextAction::Restart { .. }),
5954            "the old non-wire process's clean exit must restart"
5955        );
5956        module.drain().await.unwrap();
5957    }
5958
5959    #[cfg(unix)]
5960    #[tokio::test]
5961    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5962        use std::os::unix::fs::PermissionsExt;
5963        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5964        let script = dir.join("module.sh");
5965        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5966        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5967        let record_path = dir.join("live-children.json");
5968        let supervisor =
5969            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5970                .with_live_children_record(&record_path);
5971        for (program, args) in [
5972            (PathBuf::from("sleep"), vec!["60".into()]),
5973            (script, vec![]),
5974        ] {
5975            let spec = ModuleSpec {
5976                module_id: "image-identity".into(),
5977                program: program.clone(),
5978                args,
5979                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5980                    .into_iter()
5981                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5982                    .collect(),
5983                reserved: false,
5984                reserved_prefixes: vec![],
5985                protocol: ModuleProtocol::None,
5986                overlap: Default::default(),
5987            };
5988            let module = supervisor.spawn(spec).unwrap();
5989            #[cfg(target_os = "macos")]
5990            {
5991                // SETEXEC confirmation is asynchronous; the orphan record must
5992                // identify the final image, never the intermediate trampoline.
5993                let deadline = Instant::now() + Duration::from_secs(5);
5994                while crate::live_children::read_record(&record_path)
5995                    .unwrap()
5996                    .iter()
5997                    .all(|entry| entry.executable.is_none())
5998                {
5999                    assert!(Instant::now() < deadline, "module image was not confirmed");
6000                    tokio::time::sleep(Duration::from_millis(5)).await;
6001                }
6002            }
6003            let entry = crate::live_children::read_record(&record_path)
6004                .unwrap()
6005                .pop()
6006                .unwrap();
6007            let observed = subc_os::Process::open(entry.pid)
6008                .unwrap()
6009                .unwrap()
6010                .observe()
6011                .unwrap();
6012            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6013            module.drain().await.unwrap();
6014            assert_eq!(
6015                verdict,
6016                crate::live_children::IdentityVerdict::Matches,
6017                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6018            );
6019        }
6020    }
6021
6022    #[cfg(unix)]
6023    fn http_fixture(
6024        dir: &std::path::Path,
6025        url: &str,
6026        threshold: u32,
6027    ) -> crate::daemon_config::ConfiguredModule {
6028        let path = dir.join("subc.jsonc");
6029        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6030            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6031            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6032            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6033            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6034        }}}).to_string()).unwrap();
6035        crate::daemon_config::load(&path)
6036            .unwrap()
6037            .unwrap()
6038            .modules
6039            .pop()
6040            .unwrap()
6041    }
6042
6043    #[cfg(unix)]
6044    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6045        timeout(Duration::from_secs(5), async {
6046            loop {
6047                if module.status().unwrap().health.status == status {
6048                    break;
6049                }
6050                sleep(Duration::from_millis(5)).await;
6051            }
6052        })
6053        .await
6054        .unwrap_or_else(|_| {
6055            panic!(
6056                "expected {status:?}, got {:?}",
6057                module.status().unwrap().health
6058            )
6059        });
6060    }
6061
6062    #[cfg(unix)]
6063    #[tokio::test]
6064    async fn http_health_status_flips_ok_failing_ok() {
6065        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6066        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6067        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6068        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6069        let serving_status = status.clone();
6070        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6071        let server = tokio::spawn(async move {
6072            loop {
6073                let (mut stream, _) = listener.accept().await.unwrap();
6074                let mut request = [0u8; 2048];
6075                let count = stream.read(&mut request).await.unwrap();
6076                assert!(count > 0, "a probe must send an HTTP request");
6077                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6078                let body = if code == 200 {
6079                    "ready"
6080                } else {
6081                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6082                };
6083                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6084                let _ = stream.write_all(response.as_bytes()).await;
6085            }
6086        });
6087        let configured = http_fixture(&dir, &url, 1000);
6088        let module =
6089            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6090                .supervise_configured_with_health(
6091                    configured.module_spec(),
6092                    true,
6093                    configured.health,
6094                    configured.drain_timeout_ms,
6095                    configured.restart,
6096                )
6097                .unwrap();
6098        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6099        status.store(503, std::sync::atomic::Ordering::SeqCst);
6100        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6101        assert!(module
6102            .status()
6103            .unwrap()
6104            .health
6105            .detail
6106            .unwrap()
6107            .contains("scratch failure"));
6108        status.store(200, std::sync::atomic::Ordering::SeqCst);
6109        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6110        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6111        let before = module.status().unwrap();
6112        let (spec, mut health) = module.configuration().unwrap();
6113        health.http = None;
6114        module
6115            .update_configuration(spec.clone(), health.clone(), Some(10))
6116            .await
6117            .unwrap();
6118        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6119        health.http = Some(url);
6120        module
6121            .update_configuration(spec, health, Some(10))
6122            .await
6123            .unwrap();
6124        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6125        assert_eq!(
6126            module.status().unwrap().pid,
6127            before.pid,
6128            "changing a probe must apply live, not restart its process"
6129        );
6130        let (spec, mut health) = module.configuration().unwrap();
6131        health.failure_threshold = 2;
6132        module
6133            .update_configuration(spec, health, Some(10))
6134            .await
6135            .unwrap();
6136        status.store(503, std::sync::atomic::Ordering::SeqCst);
6137        timeout(Duration::from_secs(5), async {
6138            while module.status().unwrap().spawn_generation == before.spawn_generation {
6139                sleep(Duration::from_millis(5)).await;
6140            }
6141        })
6142        .await
6143        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6144        module.drain().await.unwrap();
6145        server.abort();
6146    }
6147
6148    #[cfg(unix)]
6149    #[tokio::test]
6150    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6151        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6152        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6153        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6154        drop(unused);
6155        let configured = http_fixture(&dir, &url, 2);
6156        let module =
6157            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6158                .supervise_configured_with_health(
6159                    configured.module_spec(),
6160                    true,
6161                    configured.health,
6162                    configured.drain_timeout_ms,
6163                    configured.restart,
6164                )
6165                .unwrap();
6166        let before = module.status().unwrap().spawn_generation;
6167        timeout(Duration::from_secs(5), async {
6168            loop {
6169                let status = module.status().unwrap();
6170                if status.spawn_generation > before {
6171                    assert!(status.lifetime_restarts > 0);
6172                    break;
6173                }
6174                sleep(Duration::from_millis(5)).await;
6175            }
6176        })
6177        .await
6178        .expect("sustained HTTP refusal must trigger the health restart policy");
6179        module.drain().await.unwrap();
6180    }
6181
6182    #[cfg(unix)]
6183    #[tokio::test]
6184    async fn http_health_timeout_honours_deadline() {
6185        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6186        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6187        let server = tokio::spawn(async move {
6188            let _held = listener.accept().await.unwrap();
6189            std::future::pending::<()>().await;
6190        });
6191        let error = timeout(
6192            Duration::from_secs(1),
6193            probe_http_health(&url, Duration::from_millis(10)),
6194        )
6195        .await
6196        .expect("the probe must enforce its own deadline")
6197        .unwrap_err();
6198        server.abort();
6199        assert!(error.to_string().contains("timed out"));
6200    }
6201
6202    #[cfg(unix)]
6203    #[tokio::test]
6204    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6205        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6206        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6207        let url = format!(
6208            "http://localhost:{}/healthz",
6209            listener.local_addr().unwrap().port()
6210        );
6211        let server = tokio::spawn(async move {
6212            let (mut stream, _) = listener.accept().await.unwrap();
6213            let mut request = [0u8; 2048];
6214            assert!(stream.read(&mut request).await.unwrap() > 0);
6215            stream
6216                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6217                .await
6218                .unwrap();
6219        });
6220        // The deadline only bounds a hang. A probe that never tried the IPv6
6221        // address would be refused on 127.0.0.1 and fail at once, so a longer
6222        // deadline does not weaken the assertion; one second timed out under a
6223        // loaded parallel test run.
6224        assert_eq!(
6225            probe_http_health(&url, Duration::from_secs(10))
6226                .await
6227                .unwrap()
6228                .status,
6229            HealthStatus::Ok
6230        );
6231        server.await.unwrap();
6232    }
6233
6234    #[cfg(unix)]
6235    #[tokio::test]
6236    async fn http_health_timeout_keeps_partial_status_and_body() {
6237        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6238        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6239        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6240        let server = tokio::spawn(async move {
6241            let (mut stream, _) = listener.accept().await.unwrap();
6242            let mut request = [0u8; 2048];
6243            assert!(stream.read(&mut request).await.unwrap() > 0);
6244            stream
6245                .write_all(
6246                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6247                )
6248                .await
6249                .unwrap();
6250            std::future::pending::<()>().await;
6251        });
6252        let error = probe_http_health(&url, Duration::from_secs(1))
6253            .await
6254            .unwrap_err()
6255            .to_string();
6256        server.abort();
6257        assert!(
6258            error.contains("timed out")
6259                && error.contains("503 Unavailable")
6260                && error.contains("partial diagnostic"),
6261            "{error}"
6262        );
6263    }
6264
6265    #[cfg(unix)]
6266    #[tokio::test]
6267    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6268        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6269        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6270        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6271        let server = tokio::spawn(async move {
6272            let (mut stream, _) = listener.accept().await.unwrap();
6273            let mut request = [0u8; 2048];
6274            assert!(stream.read(&mut request).await.unwrap() > 0);
6275            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6276            let response = format!(
6277                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6278                body.len()
6279            );
6280            stream.write_all(response.as_bytes()).await.unwrap();
6281        });
6282        let error = probe_http_health(&url, Duration::from_secs(1))
6283            .await
6284            .unwrap_err()
6285            .to_string();
6286        server.await.unwrap();
6287        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6288        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6289        assert!(!error.contains("not-in-diagnostic"));
6290    }
6291
6292    #[cfg(unix)]
6293    #[tokio::test]
6294    async fn http_health_real_nats_server_monitoring() {
6295        if std::process::Command::new("nats-server")
6296            .arg("--version")
6297            .env("XDG_DATA_HOME", std::env::temp_dir())
6298            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6299            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6300            .output()
6301            .is_err()
6302        {
6303            eprintln!(
6304                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6305            );
6306            return;
6307        }
6308        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6309        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6310        let port = monitor.local_addr().unwrap().port();
6311        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6312        let client_port = client.local_addr().unwrap().port();
6313        let config = dir.join("server.conf");
6314        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6315        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6316        configured.program = PathBuf::from("nats-server");
6317        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6318        drop(monitor);
6319        drop(client);
6320        let module =
6321            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6322                .supervise_configured_with_health(
6323                    configured.module_spec(),
6324                    true,
6325                    configured.health,
6326                    configured.drain_timeout_ms,
6327                    configured.restart,
6328                )
6329                .unwrap();
6330        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6331        module.drain().await.unwrap();
6332    }
6333
6334    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6335    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6336        let supervisor =
6337            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6338        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6339        let initial = ModuleSpec {
6340            module_id: "rescan-preserves-spawn-facts".to_string(),
6341            program: PathBuf::from("/spawned/module"),
6342            args: Vec::new(),
6343            env: Vec::new(),
6344            reserved: false,
6345            reserved_prefixes: Vec::new(),
6346            protocol: ModuleProtocol::Subc,
6347            overlap: Default::default(),
6348        };
6349        let module = supervisor.supervised_module(
6350            initial.clone(),
6351            supervisor.runtime_config(),
6352            snapshot,
6353            None,
6354        );
6355        let before = module.status().unwrap();
6356        let mut replacement = initial;
6357        replacement.program = PathBuf::from("/rescanned/replacement-module");
6358
6359        module
6360            .update_configuration(replacement, HealthConfig::default(), None)
6361            .await
6362            .unwrap();
6363
6364        let after = module.status().unwrap();
6365        assert_eq!(after.pid, before.pid);
6366        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6367        assert_eq!(after.spawned_from, before.spawned_from);
6368        drop(module);
6369    }
6370}
6371
6372fn unix_ms_now() -> u64 {
6373    SystemTime::now()
6374        .duration_since(UNIX_EPOCH)
6375        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6376        .unwrap_or(0)
6377}
6378
6379async fn supervise_loop(
6380    mut spec: ModuleSpec,
6381    mut runtime: SupervisorRuntimeConfig,
6382    registry: Arc<Registry>,
6383    process_liveness: Arc<SupervisorProcessLiveness>,
6384    snapshot: SharedSnapshot,
6385    mut child: Option<SupervisedChild>,
6386    mut commands: mpsc::Receiver<SupervisorCommand>,
6387) {
6388    let mut health_probe = HealthProbeRuntime::default();
6389    // Registry writes (including embedded callers) notify this module only.
6390    // Subscribe before the first refresh; watch retains changes that arrive
6391    // while commands or probes are running. No polling fallback is needed.
6392    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6393        Ok(changes) => Some(changes),
6394        Err(err) => {
6395            // A poisoned registry cannot accept further writes, so it cannot
6396            // recover via a timer. Keep exit and command handling alive.
6397            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6398            None
6399        }
6400    };
6401    // All restart backoffs run here, including health and operator requests.
6402    // While one is pending the loop serves commands, so disable or drain can
6403    // cancel the replacement without spawning a process just to stop it.
6404    let mut pending_respawn: Option<PendingRespawn> = None;
6405    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6406    // before anything else so a stop that interrupted a swap runs at once.
6407    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6408    let mut previous_instance_recheck = tokio::time::interval_at(
6409        Instant::now() + Duration::from_secs(5),
6410        Duration::from_secs(5),
6411    );
6412    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6413    loop {
6414        #[cfg(test)]
6415        {
6416            lock_snapshot(&snapshot).unwrap().actor_select = None;
6417        }
6418        #[cfg(target_os = "macos")]
6419        if let Some(active) = child.as_mut() {
6420            active.confirm_privacy_exec().await;
6421        }
6422        if let Some(scheduled) = runtime
6423            .scheduled_respawn
6424            .lock()
6425            .unwrap_or_else(|p| p.into_inner())
6426            .take()
6427        {
6428            pending_respawn = Some(scheduled);
6429        }
6430        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6431            pending_respawn = None;
6432            cancel_deferred_reload(
6433                &runtime,
6434                &spec.module_id,
6435                "respawn cancelled by a supervisor command",
6436            );
6437        }
6438        if child.is_none()
6439            && pending_respawn.is_none()
6440            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6441        {
6442            cancel_deferred_reload(
6443                &runtime,
6444                &spec.module_id,
6445                "respawn cancelled before a replacement was spawned",
6446            );
6447            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6448                state.respawn_pending = false;
6449                state.coalesced_restart_pending = false;
6450                if matches!(
6451                    state.state,
6452                    ModuleState::Restarting
6453                        | ModuleState::Starting
6454                        | ModuleState::Draining
6455                        | ModuleState::Unresponsive
6456                ) {
6457                    error!(module_id = %spec.module_id, state = ?state.state,
6458                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6459                    state.state = ModuleState::Failed;
6460                    clear_current_process_facts(state);
6461                }
6462            });
6463        }
6464        if let Some(command) = requeued.pop_front() {
6465            if !handle_supervisor_command(
6466                command,
6467                &mut spec,
6468                &mut runtime,
6469                &registry,
6470                &process_liveness,
6471                &snapshot,
6472                &mut child,
6473                &mut commands,
6474                &mut requeued,
6475            )
6476            .await
6477            {
6478                return;
6479            }
6480            if child.is_some() || !respawn_still_pending(&snapshot) {
6481                pending_respawn = None;
6482                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6483                    state.respawn_pending = false
6484                });
6485            }
6486            continue;
6487        }
6488        if child.is_some() {
6489            #[cfg(test)]
6490            {
6491                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6492            }
6493            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6494            let wake_after = health_probe.wake_after();
6495            let probe_sleep = async move {
6496                match wake_after {
6497                    Some(delay) => sleep(delay).await,
6498                    None => std::future::pending().await,
6499                }
6500            };
6501            tokio::pin!(probe_sleep);
6502            // A `protocol: "none"` child (a plain process that never registers
6503            // over the subc wire, such as nats-server) with no HTTP health
6504            // check has nothing to probe, so it waits only for its exit or a
6505            // supervisor command. A rescan that adds an HTTP check is a command.
6506            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6507            #[cfg(test)]
6508            {
6509                let registration_pending = wire_child
6510                    && registration_changes
6511                        .as_ref()
6512                        .is_some_and(|changes| changes.has_changed().unwrap());
6513                let mut state = lock_snapshot(&snapshot).unwrap();
6514                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6515                    generation: state.spawn_generation,
6516                    turn: state.actor_turns,
6517                    registered_connection: health_probe.registered_connection,
6518                    next_probe_at: health_probe.next_probe_at,
6519                    wake_after,
6520                });
6521            }
6522            let active_child = child.as_mut().expect("child checked above");
6523            tokio::select! {
6524                wait_result = active_child.wait() => {
6525                    #[cfg(test)]
6526                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6527                    // Every arm below that gives up on the CHILD must keep the
6528                    // supervision task itself alive (child = None, loop
6529                    // continues into command-serving mode). Returning here
6530                    // closes the command channel, which makes the module
6531                    // permanently unrestartable in-band: a clean child exit
6532                    // of an enabled module once wedged the fleet this way
6533                    // ('supervisor command channel is closed') and required a
6534                    // full daemon restart to recover.
6535                    let exit_report = match wait_result {
6536                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6537                        Err(err) => {
6538                            active_child.drain_stderr(&spec.module_id).await;
6539                            fail_after_wait_error(
6540                                &spec.module_id,
6541                                &snapshot,
6542                                &runtime.terminal_ring,
6543                                &runtime.spawn_events,
6544                                format!("failed to wait for supervised module: {err}"),
6545                            );
6546                            untrack_if_registration_released(
6547                                &process_liveness,
6548                                &registry,
6549                                &spec.module_id,
6550                                &snapshot,
6551                            );
6552                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6553                            child = None;
6554                            continue;
6555                        }
6556                    };
6557                    active_child.drain_stderr(&spec.module_id).await;
6558
6559                    let next = on_child_exit(
6560                        &spec,
6561                        runtime.restart_policy,
6562                        &registry,
6563                        &snapshot,
6564                        &runtime.terminal_ring,
6565                        &runtime.spawn_events,
6566                        &runtime.child_roster,
6567                        exit_report,
6568                    ).await;
6569                    runtime.start_requester = None;
6570                    runtime.pending_activity_restart = false;
6571                    // The exit is recorded, so a daemon shutdown may stop
6572                    // waiting for this child (see `SupervisedChild::wait`).
6573                    active_child.release_roster();
6574                    match next {
6575                        NextAction::Stop { registration_released } => {
6576                            if registration_released {
6577                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6578                            }
6579                            child = None;
6580                        }
6581                        NextAction::Restart { schedule } => {
6582                            let delay = schedule.map_or(
6583                                runtime.restart_policy.delay_for_restart(0),
6584                                |schedule| schedule.delay,
6585                            );
6586                            if let Some(schedule) = schedule {
6587                                log_crash_respawn(&spec.module_id, schedule);
6588                            }
6589                            // The exited child is fully recorded at this point,
6590                            // so release it and count the backoff down in the
6591                            // command-serving branch below rather than sleeping
6592                            // here: commands cannot be received from inside this
6593                            // select arm, and an operator disable or drain that
6594                            // arrives during the backoff must cancel the pending
6595                            // respawn instead of waiting for it to spawn first.
6596                            child = None;
6597                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6598                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6599                        }
6600                    }
6601                }
6602                command = commands.recv() => {
6603                    #[cfg(test)]
6604                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6605                    let Some(command) = command else {
6606                        return;
6607                    };
6608                    if !handle_supervisor_command(
6609                        command,
6610                        &mut spec,
6611                        &mut runtime,
6612                        &registry,
6613                        &process_liveness,
6614                        &snapshot,
6615                        &mut child,
6616                        &mut commands,
6617                        &mut requeued,
6618                    ).await {
6619                        return;
6620                    }
6621                }
6622                _ = async {
6623                    match registration_changes.as_mut() {
6624                        Some(changes) => { let _ = changes.changed().await; }
6625                        None => std::future::pending().await,
6626                    }
6627                }, if wire_child => {
6628                    #[cfg(test)]
6629                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6630                }
6631                _ = &mut probe_sleep => {
6632                    #[cfg(test)]
6633                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6634                    if health_probe.due() {
6635                        run_health_probe_cycle(
6636                            &spec,
6637                            &runtime,
6638                            &registry,
6639                            &process_liveness,
6640                            &snapshot,
6641                            &mut child,
6642                        ).await;
6643                        if child.is_none() {
6644                            runtime.start_requester = None;
6645                            runtime.pending_activity_restart = false;
6646                        }
6647                        if child.is_some() {
6648                            health_probe.schedule_next(&spec, runtime.health.cadence);
6649                        }
6650                    }
6651                }
6652            }
6653        } else if let Some(pending) = pending_respawn {
6654            tokio::select! {
6655                _ = sleep_until(pending.deadline) => {
6656                    pending_respawn = None;
6657                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6658                    // A command handled below while the backoff elapsed may
6659                    // have stopped the module; never respawn past an operator's
6660                    // disable or drain.
6661                    if !respawn_still_pending(&snapshot) {
6662                        continue;
6663                    }
6664                    // The daemon began shutting down during the backoff: the
6665                    // spawn would be refused anyway, and refusing it here
6666                    // leaves the module stopped instead of reporting a
6667                    // failed restart.
6668                    if runtime.child_roster.is_closed() {
6669                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6670                            state.state = ModuleState::Stopped;
6671                        });
6672                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6673                        continue;
6674                    }
6675                    if let Err(err) = release_dead_registration(
6676                        &registry,
6677                        runtime.forwarding.as_deref(),
6678                        &snapshot,
6679                        &spec.module_id,
6680                    ).await {
6681                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6682                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6683                        continue;
6684                    }
6685
6686                    if matches!(pending.kind, RespawnKind::Reload) {
6687                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6688                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6689                        if result.is_ok() && runtime.pending_activity_restart {
6690                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6691                        }
6692                        runtime.pending_activity_restart = false;
6693                        if let Some(reply) = reply { let _ = reply.send(result); }
6694                        continue;
6695                    }
6696                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6697                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6698                        Ok(next_child) => {
6699                            child = Some(next_child);
6700                            if runtime.pending_activity_restart {
6701                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6702                                runtime.pending_activity_restart = false;
6703                            }
6704                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6705                        }
6706                        Err(err) => {
6707                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6708                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6709                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6710                        }
6711                    }
6712                }
6713                command = commands.recv() => {
6714                    let Some(command) = command else {
6715                        return;
6716                    };
6717                    if !handle_supervisor_command(
6718                        command,
6719                        &mut spec,
6720                        &mut runtime,
6721                        &registry,
6722                        &process_liveness,
6723                        &snapshot,
6724                        &mut child,
6725                        &mut commands,
6726                        &mut requeued,
6727                    ).await {
6728                        return;
6729                    }
6730                    // Reconcile the pending respawn with what the command did:
6731                    // a start may already have spawned a fresh child,
6732                    // while a disable or drain moved the snapshot out of the
6733                    // state the respawn was counting down from.
6734                    if child.is_some() || !respawn_still_pending(&snapshot) {
6735                        pending_respawn = None;
6736                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6737                    }
6738                }
6739            }
6740        } else {
6741            let command = if lock_snapshot(&snapshot)
6742                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6743            {
6744                tokio::select! {
6745                    command = commands.recv() => command,
6746                    _ = previous_instance_recheck.tick() => {
6747                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6748                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6749                            if enabled && !runtime.child_roster.is_closed() {
6750                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6751                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6752                                    Ok(next_child) => child = Some(next_child),
6753                                    Err(error) => {
6754                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6755                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6756                                    }
6757                                }
6758                            } else {
6759                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6760                                    state.waiting_for_previous_pid = None;
6761                                    state.waiting_for_previous_detail = None;
6762                                });
6763                            }
6764                        }
6765                        continue;
6766                    }
6767                }
6768            } else {
6769                commands.recv().await
6770            };
6771            let Some(command) = command else {
6772                return;
6773            };
6774            if !handle_supervisor_command(
6775                command,
6776                &mut spec,
6777                &mut runtime,
6778                &registry,
6779                &process_liveness,
6780                &snapshot,
6781                &mut child,
6782                &mut commands,
6783                &mut requeued,
6784            )
6785            .await
6786            {
6787                return;
6788            }
6789        }
6790    }
6791}
6792
6793fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6794    info!(
6795        module_id,
6796        restart_in_window = schedule.restart_in_window,
6797        delay_ms = schedule.delay.as_millis() as u64,
6798        "respawning after crash"
6799    );
6800}
6801
6802/// Whether the respawn a backoff was counting down to is still wanted. A
6803/// disable or drain handled while the backoff elapsed moves the snapshot out
6804/// of `Restarting`, and the operator's stop must win over the pending respawn,
6805/// so every sleep-then-spawn path re-validates against the live snapshot
6806/// instead of assuming the state it left behind still holds.
6807fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6808    matches!(
6809        lock_snapshot(snapshot),
6810        Ok(state) if state.enabled && state.state == ModuleState::Restarting
6811    )
6812}
6813
6814enum NextAction {
6815    Stop {
6816        registration_released: bool,
6817    },
6818    Restart {
6819        schedule: Option<CrashRestartSchedule>,
6820    },
6821}
6822
6823#[allow(clippy::too_many_arguments)]
6824async fn handle_supervisor_command(
6825    command: SupervisorCommand,
6826    spec: &mut ModuleSpec,
6827    runtime: &mut SupervisorRuntimeConfig,
6828    registry: &Arc<Registry>,
6829    process_liveness: &SupervisorProcessLiveness,
6830    snapshot: &SharedSnapshot,
6831    child: &mut Option<SupervisedChild>,
6832    commands: &mut mpsc::Receiver<SupervisorCommand>,
6833    requeued: &mut VecDeque<SupervisorCommand>,
6834) -> bool {
6835    let command =
6836        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6837            match command {
6838                SupervisorCommand::Restart { reply, .. }
6839                | SupervisorCommand::Reload { reply, .. }
6840                | SupervisorCommand::Swap { reply, .. } => {
6841                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6842                    return true;
6843                }
6844                SupervisorCommand::SetEnabled {
6845                    enabled: true,
6846                    reply,
6847                    ..
6848                } => {
6849                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6850                    return true;
6851                }
6852                other => other,
6853            }
6854        } else {
6855            command
6856        };
6857    match command {
6858        SupervisorCommand::Drain { requester, reply } => {
6859            // A plain stop runs no forwarding drain, so nothing reaches the
6860            // module over its connection before the wait: ask by signal.
6861            let result = drain_optional_child(
6862                &spec.module_id,
6863                spec.protocol,
6864                StopNotice::NotSent,
6865                registry,
6866                runtime.forwarding.as_deref(),
6867                snapshot,
6868                &runtime.terminal_ring,
6869                &runtime.spawn_events,
6870                child,
6871                runtime.drain_timeout,
6872                ModuleState::Stopped,
6873                None,
6874            )
6875            .await;
6876            let registration_released = result.is_ok();
6877            if registration_released {
6878                runtime.record_activity_transition(
6879                    "subc.module_stopped",
6880                    &spec.module_id,
6881                    requester,
6882                );
6883            }
6884            let _ = reply.send(result);
6885            if registration_released {
6886                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6887            }
6888            false
6889        }
6890        SupervisorCommand::Retire { requester, reply } => {
6891            let result = async {
6892                let stop_notice = begin_forwarding_drain_if_configured(
6893                    spec,
6894                    runtime,
6895                    registry,
6896                    snapshot,
6897                    None,
6898                    RouteCloseReason::Disable,
6899                )
6900                .await?;
6901                drain_optional_child(
6902                    &spec.module_id,
6903                    spec.protocol,
6904                    stop_notice,
6905                    registry,
6906                    runtime.forwarding.as_deref(),
6907                    snapshot,
6908                    &runtime.terminal_ring,
6909                    &runtime.spawn_events,
6910                    child,
6911                    runtime.drain_timeout,
6912                    ModuleState::Stopped,
6913                    None,
6914                )
6915                .await
6916            }
6917            .await;
6918            let registration_released = result.is_ok();
6919            if registration_released {
6920                runtime.record_activity_transition(
6921                    "subc.module_stopped",
6922                    &spec.module_id,
6923                    requester,
6924                );
6925            }
6926            let _ = reply.send(result);
6927            if registration_released {
6928                process_liveness.untrack_if_current(&spec.module_id, snapshot);
6929            }
6930            false
6931        }
6932        SupervisorCommand::Restart {
6933            requester,
6934            drain_timeout_ms,
6935            received_at_generation,
6936            queued_at,
6937            reply,
6938        } => {
6939            // Without this line a restart that waited in the queue (behind a
6940            // health probe cycle or another command) was invisible: the log
6941            // showed only the drain timing out, minutes after the operator's call.
6942            info!(
6943                module_id = %spec.module_id,
6944                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6945                "restart command dequeued"
6946            );
6947            // ACK AT INITIATION, not completion. The blocking form deadlocked any
6948            // caller whose own request lane rides the module being restarted: the
6949            // caller's in-flight request keeps the drain from quiescing, the drain
6950            // keeps the restart from completing, and the completion keeps the reply
6951            // from releasing the caller — so the drain always timed out and cut the
6952            // initiator with a GOODBYE, even on a healthy module. Replying once the
6953            // restart is validated lets a self-lane caller settle, which is exactly
6954            // what makes the drain succeed. Completion is observable via
6955            // supervisor.list / module status; a post-ack failure lands the module
6956            // in a visible terminal state below rather than in a reply nobody can
6957            // receive.
6958            let validation = match lock_snapshot(snapshot) {
6959                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6960                    module_id: spec.module_id.clone(),
6961                }),
6962                Ok(_) => Ok(()),
6963                Err(err) => Err(err),
6964            };
6965            let initiated = validation.is_ok();
6966            let _ = reply.send(validation);
6967            // A restart asks for a fresh process. Commands run one at a time,
6968            // so a restart queued behind another restart (two operator calls
6969            // in quick succession) is dequeued the moment the first one has
6970            // spawned its replacement -- before that process has sent HELLO.
6971            // Running it would drain and kill the process the first restart
6972            // just produced, which is the opposite of what both callers asked
6973            // for. If a process spawned after this request was received is
6974            // still supervised, the request is already satisfied. Not when the
6975            // configuration changed since that spawn: then the newer process
6976            // predates the spec this restart may exist to apply.
6977            // Generation publication can lag a child that is already serving
6978            // while its identity is recorded. Only process creation after request
6979            // receipt satisfies a restart, not publication after receipt.
6980            let satisfied_by_generation = if initiated
6981                && child
6982                    .as_ref()
6983                    .is_some_and(|child| child.spawned_at > queued_at)
6984            {
6985                lock_snapshot(snapshot).ok().and_then(|state| {
6986                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
6987                })
6988            } else {
6989                None
6990            };
6991            let satisfied_by_pending = initiated
6992                && child.is_none()
6993                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6994                    // No child exists yet, so this replacement will necessarily
6995                    // be created after the queued request, unlike a published
6996                    // generation whose process may already have been serving.
6997                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6998                    if pending {
6999                        state.coalesced_restart_pending = true;
7000                    }
7001                    pending
7002                });
7003            if satisfied_by_pending {
7004                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7005            } else if let Some(generation) = satisfied_by_generation {
7006                info!(
7007                    module_id = %spec.module_id,
7008                    received_at_generation,
7009                    "restart already satisfied by generation {generation}; not restarting again"
7010                );
7011            } else if initiated {
7012                // Precedence: this restart's operator override, else the module's
7013                // configured budget (already resolved into the runtime).
7014                let drain_timeout = drain_timeout_ms
7015                    .map(Duration::from_millis)
7016                    .unwrap_or(runtime.drain_timeout);
7017                if let Err(err) = restart_child(
7018                    spec,
7019                    runtime,
7020                    registry,
7021                    process_liveness,
7022                    snapshot,
7023                    child,
7024                    drain_timeout,
7025                )
7026                .await
7027                {
7028                    warn!(
7029                        module_id = %spec.module_id,
7030                        error = %err,
7031                        "operator restart failed after initiation ack; module state carries the outcome"
7032                    );
7033                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7034                        state.state = ModuleState::Failed;
7035                        clear_current_process_facts(state);
7036                    });
7037                } else {
7038                    runtime.start_requester = requester;
7039                    runtime.pending_activity_restart = true;
7040                }
7041            }
7042            true
7043        }
7044        SupervisorCommand::Reload { requester, reply } => {
7045            let result =
7046                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7047            if result.is_ok() {
7048                runtime.start_requester = requester;
7049                runtime.pending_activity_restart = true;
7050            }
7051            if result.is_ok()
7052                && runtime
7053                    .scheduled_respawn
7054                    .lock()
7055                    .unwrap_or_else(|p| p.into_inner())
7056                    .as_ref()
7057                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7058            {
7059                *runtime
7060                    .deferred_reload_reply
7061                    .lock()
7062                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7063            } else {
7064                let _ = reply.send(result);
7065            }
7066            true
7067        }
7068        SupervisorCommand::SetEnabled {
7069            enabled,
7070            requester,
7071            reply,
7072        } => {
7073            let previous_requester = runtime.start_requester.clone();
7074            if enabled {
7075                runtime.start_requester = requester.clone();
7076            }
7077            let result = set_child_enabled(
7078                spec,
7079                runtime,
7080                registry,
7081                process_liveness,
7082                snapshot,
7083                child,
7084                enabled,
7085            )
7086            .await;
7087            if matches!(result, Ok(true)) {
7088                runtime.pending_activity_restart = false;
7089                if enabled {
7090                    runtime.start_requester = requester;
7091                } else {
7092                    runtime.start_requester = None;
7093                    runtime.record_activity_transition(
7094                        "subc.module_disabled",
7095                        &spec.module_id,
7096                        requester,
7097                    );
7098                }
7099            } else {
7100                runtime.start_requester = previous_requester;
7101            }
7102            let _ = reply.send(result);
7103            true
7104        }
7105        SupervisorCommand::UpdateConfiguration {
7106            spec: next_spec,
7107            health,
7108            drain_timeout_ms,
7109            reply,
7110        } => {
7111            if let Some(handle) = &runtime.supervisor_handle {
7112                handle.apply_identity_configuration(&next_spec);
7113            }
7114            *spec = next_spec;
7115            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7116                state.configuration_updated_since_spawn = true;
7117            });
7118            let health_changed = runtime.health != health;
7119            runtime.health = health;
7120            // Reset the cadence and old endpoint's failure streak on a live
7121            // health-policy change rather than waiting for its old deadline.
7122            if health_changed {
7123                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7124                    state.health = ModuleHealthStatus::default();
7125                });
7126            }
7127            runtime.drain_timeout = drain_timeout_ms
7128                .map(Duration::from_millis)
7129                .unwrap_or(runtime.default_drain_timeout);
7130            *runtime
7131                .effective_drain_timeout
7132                .lock()
7133                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7134            let _ = reply.send(());
7135            true
7136        }
7137        SupervisorCommand::Swap {
7138            requester,
7139            ready_timeout,
7140            reply,
7141        } => {
7142            let mut swap_runtime = runtime.clone();
7143            swap_runtime.start_requester = requester;
7144            let before = lock_snapshot(snapshot)
7145                .ok()
7146                .map(|state| state.spawn_generation);
7147            let end = swap::run_swap(
7148                spec,
7149                &swap_runtime,
7150                registry,
7151                process_liveness,
7152                snapshot,
7153                child,
7154                commands,
7155                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7156                reply,
7157            )
7158            .await;
7159            if before
7160                != lock_snapshot(snapshot)
7161                    .ok()
7162                    .map(|state| state.spawn_generation)
7163            {
7164                runtime.start_requester = swap_runtime.start_requester;
7165            }
7166            requeued.extend(end.requeue);
7167            true
7168        }
7169    }
7170}
7171
7172async fn restart_child(
7173    spec: &ModuleSpec,
7174    runtime: &SupervisorRuntimeConfig,
7175    registry: &Registry,
7176    process_liveness: &SupervisorProcessLiveness,
7177    snapshot: &SharedSnapshot,
7178    child: &mut Option<SupervisedChild>,
7179    drain_timeout: Duration,
7180) -> Result<(), SuperviseError> {
7181    // Restart cycles a running module; it must not silently start a disabled one.
7182    if !lock_snapshot(snapshot)?.enabled {
7183        return Err(SuperviseError::Disabled {
7184            module_id: spec.module_id.clone(),
7185        });
7186    }
7187    let stop_notice = begin_forwarding_drain_with_timeout(
7188        spec,
7189        runtime,
7190        registry,
7191        snapshot,
7192        None,
7193        RouteCloseReason::Restart,
7194        drain_timeout,
7195    )
7196    .await?;
7197
7198    if child.is_some() {
7199        drain_optional_child(
7200            &spec.module_id,
7201            spec.protocol,
7202            stop_notice,
7203            registry,
7204            runtime.forwarding.as_deref(),
7205            snapshot,
7206            &runtime.terminal_ring,
7207            &runtime.spawn_events,
7208            child,
7209            drain_timeout,
7210            ModuleState::Restarting,
7211            Some(true),
7212        )
7213        .await?;
7214    } else {
7215        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7216            state.enabled = true;
7217            state.state = ModuleState::Restarting;
7218            clear_current_process_facts(state);
7219        })?;
7220        release_dead_registration(
7221            registry,
7222            runtime.forwarding.as_deref(),
7223            snapshot,
7224            &spec.module_id,
7225        )
7226        .await?;
7227    }
7228
7229    reset_restart_count(snapshot, &spec.module_id)?;
7230    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7231    schedule_respawn(
7232        runtime,
7233        snapshot,
7234        &spec.module_id,
7235        runtime.restart_policy.backoff,
7236        RespawnKind::Spawn,
7237    )
7238}
7239
7240async fn reload_child(
7241    spec: &ModuleSpec,
7242    runtime: &SupervisorRuntimeConfig,
7243    registry: &Registry,
7244    process_liveness: &SupervisorProcessLiveness,
7245    snapshot: &SharedSnapshot,
7246    child: &mut Option<SupervisedChild>,
7247) -> Result<(), SuperviseError> {
7248    // Reload cycles a running module; it must not silently start a disabled one.
7249    if !lock_snapshot(snapshot)?.enabled {
7250        return Err(SuperviseError::Disabled {
7251            module_id: spec.module_id.clone(),
7252        });
7253    }
7254    let stop_notice = begin_forwarding_drain(
7255        spec,
7256        runtime,
7257        registry,
7258        snapshot,
7259        Some(true),
7260        RouteCloseReason::Reload,
7261    )
7262    .await?;
7263
7264    if child.is_some() {
7265        drain_optional_child(
7266            &spec.module_id,
7267            spec.protocol,
7268            stop_notice,
7269            registry,
7270            runtime.forwarding.as_deref(),
7271            snapshot,
7272            &runtime.terminal_ring,
7273            &runtime.spawn_events,
7274            child,
7275            runtime.drain_timeout,
7276            ModuleState::Restarting,
7277            Some(true),
7278        )
7279        .await?;
7280    } else {
7281        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7282            state.enabled = true;
7283            state.state = ModuleState::Restarting;
7284            clear_current_process_facts(state);
7285        })?;
7286        release_dead_registration(
7287            registry,
7288            runtime.forwarding.as_deref(),
7289            snapshot,
7290            &spec.module_id,
7291        )
7292        .await?;
7293    }
7294
7295    reset_restart_count(snapshot, &spec.module_id)?;
7296    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7297    schedule_respawn(
7298        runtime,
7299        snapshot,
7300        &spec.module_id,
7301        runtime.restart_policy.backoff,
7302        RespawnKind::Reload,
7303    )
7304}
7305
7306async fn finish_reload_child(
7307    spec: &ModuleSpec,
7308    runtime: &SupervisorRuntimeConfig,
7309    registry: &Registry,
7310    process_liveness: &SupervisorProcessLiveness,
7311    snapshot: &SharedSnapshot,
7312    child: &mut Option<SupervisedChild>,
7313) -> Result<(), SuperviseError> {
7314    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7315    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7316        Ok(next_child) => next_child,
7317        Err(err) => {
7318            return handle_reload_spawn_failure(
7319                spec,
7320                runtime,
7321                process_liveness,
7322                snapshot,
7323                child,
7324                format!("new child failed to spawn: {err}"),
7325            )
7326            .await;
7327        }
7328    };
7329    *child = Some(next_child);
7330
7331    let wait_outcome = {
7332        let active_child = child.as_mut().expect("new reload child was just stored");
7333        wait_for_registration_after_reload(
7334            registry,
7335            &spec.module_id,
7336            snapshot,
7337            active_child,
7338            REGISTRY_RELEASE_TIMEOUT,
7339        )
7340        .await?
7341    };
7342
7343    match wait_outcome {
7344        RegistrationWaitOutcome::Registered => {
7345            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7346            Ok(())
7347        }
7348        RegistrationWaitOutcome::Exited(exit_report) => {
7349            if let Some(active_child) = child.as_mut() {
7350                active_child.drain_stderr(&spec.module_id).await;
7351            }
7352            // Keep the reaped child's roster guard until its terminal is written.
7353            // Shutdown waits on that guard, not on the child Option used for respawn.
7354            let mut exited_child = child.take().expect("exited reload child is still stored");
7355            #[cfg(test)]
7356            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7357                gate.reached.notify_one();
7358                gate.resume.notified().await;
7359            }
7360            let result = handle_reload_child_registration_failure(
7361                spec,
7362                runtime,
7363                registry,
7364                process_liveness,
7365                snapshot,
7366                child,
7367                ReloadRegistrationFailure {
7368                    exit_report: registration_failure_exit_report(exit_report),
7369                    reason: exited_child
7370                        .spawn_failure
7371                        .clone()
7372                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7373                },
7374            )
7375            .await;
7376            exited_child.release_roster();
7377            result
7378        }
7379        RegistrationWaitOutcome::TimedOut => {
7380            let mut timed_out_child = child
7381                .take()
7382                .expect("timed-out reload child is still running");
7383            timed_out_child
7384                .start_kill()
7385                .map_err(|source| SuperviseError::Kill {
7386                    module_id: spec.module_id.clone(),
7387                    source,
7388                })?;
7389            let status = timed_out_child
7390                .wait()
7391                .await
7392                .map_err(|source| SuperviseError::Wait {
7393                    module_id: spec.module_id.clone(),
7394                    source,
7395                })?;
7396            timed_out_child.drain_stderr(&spec.module_id).await;
7397            handle_reload_child_registration_failure(
7398                spec,
7399                runtime,
7400                registry,
7401                process_liveness,
7402                snapshot,
7403                child,
7404                ReloadRegistrationFailure {
7405                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7406                        snapshot,
7407                        &timed_out_child,
7408                        &status,
7409                    )),
7410                    reason: format!(
7411                        "new child did not register within {:?}",
7412                        REGISTRY_RELEASE_TIMEOUT
7413                    ),
7414                },
7415            )
7416            .await
7417        }
7418    }
7419}
7420
7421async fn set_child_enabled(
7422    spec: &ModuleSpec,
7423    runtime: &SupervisorRuntimeConfig,
7424    registry: &Registry,
7425    process_liveness: &SupervisorProcessLiveness,
7426    snapshot: &SharedSnapshot,
7427    child: &mut Option<SupervisedChild>,
7428    enabled: bool,
7429) -> Result<bool, SuperviseError> {
7430    let (current_enabled, current_state, respawn_pending) = {
7431        let state = lock_snapshot(snapshot)?;
7432        (state.enabled, state.state, state.respawn_pending)
7433    };
7434    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7435    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7436    // clean (Stopped) has no live process and no other in-band recovery — the
7437    // operator's start is the explicit recovery act and resets the budget. Without
7438    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7439    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7440    // the one providing every agent's shell.
7441    let revive_terminal = enabled
7442        && current_enabled
7443        && child.is_none()
7444        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7445            || (current_state == ModuleState::Restarting && !respawn_pending));
7446    if current_enabled == enabled && !revive_terminal {
7447        return Ok(false);
7448    }
7449
7450    if enabled {
7451        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7452            state.enabled = true;
7453            state.state = ModuleState::Starting;
7454            clear_current_process_facts(state);
7455        })?;
7456        #[cfg(test)]
7457        if runtime.test_seed_stale_facts_before_enable_spawn {
7458            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7459                state.process_alive = true;
7460                state.pid = Some(41);
7461                state.spawned_at_ms = Some(42);
7462                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7463                state.spawned_file_identity = Some(SpawnedFileIdentity {
7464                    device: 43,
7465                    inode: 44,
7466                    #[cfg(windows)]
7467                    windows_image: None,
7468                });
7469            })?;
7470        }
7471        release_dead_registration(
7472            registry,
7473            runtime.forwarding.as_deref(),
7474            snapshot,
7475            &spec.module_id,
7476        )
7477        .await?;
7478        reset_restart_count(snapshot, &spec.module_id)?;
7479        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7480        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7481            Ok(next_child) => next_child,
7482            Err(err) => {
7483                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7484                    state.state = ModuleState::Failed;
7485                    clear_current_process_facts(state);
7486                }) {
7487                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7488                }
7489                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7490                return Err(err);
7491            }
7492        };
7493        *child = Some(next_child);
7494        debug!(module_id = %spec.module_id, "supervised module enabled");
7495        Ok(true)
7496    } else {
7497        let stop_notice = begin_forwarding_drain_if_configured(
7498            spec,
7499            runtime,
7500            registry,
7501            snapshot,
7502            Some(false),
7503            RouteCloseReason::Disable,
7504        )
7505        .await?;
7506        drain_optional_child(
7507            &spec.module_id,
7508            spec.protocol,
7509            stop_notice,
7510            registry,
7511            runtime.forwarding.as_deref(),
7512            snapshot,
7513            &runtime.terminal_ring,
7514            &runtime.spawn_events,
7515            child,
7516            runtime.drain_timeout,
7517            ModuleState::Disabled,
7518            Some(false),
7519        )
7520        .await?;
7521        debug!(module_id = %spec.module_id, "supervised module disabled");
7522        Ok(true)
7523    }
7524}
7525
7526#[allow(clippy::too_many_arguments)]
7527async fn on_child_exit(
7528    spec: &ModuleSpec,
7529    policy: RestartPolicy,
7530    registry: &Registry,
7531    snapshot: &SharedSnapshot,
7532    terminal_ring: &Arc<Mutex<TerminalRing>>,
7533    spawn_events: &SpawnEventFeed,
7534    roster: &ChildRoster,
7535    exit_report: ExitReport,
7536) -> NextAction {
7537    // Once the daemon has begun shutting down, no exit is a crash to recover
7538    // from: the module is exiting because the daemon is going away (EOF on its
7539    // connection, or a service manager signalling the whole cgroup). Record it
7540    // as such and never schedule a respawn, which would only start a process
7541    // for the shutdown to end again.
7542    if roster.is_closed() {
7543        return on_child_exit_during_daemon_shutdown(
7544            spec,
7545            registry,
7546            snapshot,
7547            terminal_ring,
7548            spawn_events,
7549            exit_report,
7550        )
7551        .await;
7552    }
7553    // Every stop the supervisor itself asks for (operator stop, disable,
7554    // restart, reload, swap, a health restart, a drain that runs out of budget)
7555    // takes the child out of the supervise loop and reaps it in
7556    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7557    // that reaches this point was not requested by the daemon.
7558    //
7559    // For a subc-wire module a clean exit is still a stop: those modules are
7560    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7561    // crash, and exiting 0 is a deliberate choice the module made. A
7562    // `protocol: "none"` module is a stock program we cannot change, and many
7563    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7564    // stop would leave the module down for good after any stray signal, so it
7565    // goes through the crash path instead: it spends restart budget, respawns
7566    // with the crash backoff, and ends `failed` when the budget runs out.
7567    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7568        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7569    match exit_report.kind {
7570        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7571            info!(
7572                module_id = %spec.module_id,
7573                exit_code = ?exit_report.code,
7574                exit_signal = ?exit_report.signal,
7575                "supervised module exited cleanly"
7576            );
7577            if let Err(err) = publish_after_terminal_record(
7578                &spec.module_id,
7579                || {
7580                    record_terminal(
7581                        &spec.module_id,
7582                        terminal_ring,
7583                        spawn_events,
7584                        &exit_report,
7585                        TerminalDisposition::Stopped,
7586                    )
7587                },
7588                || {
7589                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7590                        state.state = ModuleState::Stopped;
7591                        clear_current_process_facts(state);
7592                        state.last_exit = Some(exit_report.clone());
7593                    })
7594                },
7595            ) {
7596                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7597            }
7598            let registration_released = match wait_for_registration_release(
7599                registry,
7600                &spec.module_id,
7601                REGISTRY_RELEASE_TIMEOUT,
7602            )
7603            .await
7604            {
7605                Ok(()) => true,
7606                Err(err) => {
7607                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7608                    false
7609                }
7610            };
7611            NextAction::Stop {
7612                registration_released,
7613            }
7614        }
7615        ExitKind::Clean | ExitKind::Crash => {
7616            if unrequested_clean_exit_of_protocol_none {
7617                warn!(
7618                    module_id = %spec.module_id,
7619                    exit_code = ?exit_report.code,
7620                    exit_signal = ?exit_report.signal,
7621                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7622                );
7623            } else {
7624                warn!(
7625                    module_id = %spec.module_id,
7626                    exit_code = ?exit_report.code,
7627                    exit_signal = ?exit_report.signal,
7628                    "supervised module exited abnormally (crash)"
7629                );
7630            }
7631            let mut restart_schedule = None;
7632            let mut disposition = TerminalDisposition::Disabled;
7633            let mut final_state = ModuleState::Disabled;
7634            // Set only when the budget is what stopped the module, so the
7635            // terminal record says which limit was hit rather than leaving
7636            // `failed` to be read as "crashed once, badly".
7637            let mut disposition_detail = lock_snapshot(snapshot)
7638                .ok()
7639                .and_then(|mut state| state.spawn_failure.take());
7640            let now = Instant::now();
7641            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7642                if state.enabled {
7643                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7644                        // Reserving the last slot exhausts the budget for the
7645                        // replacement, not for this exit. Publish its scheduled
7646                        // recovery atomically with the reservation so connection
7647                        // cleanup sees it while terminal recording still holds
7648                        // the visible state at Running.
7649                        state.respawn_pending = true;
7650                        final_state = ModuleState::Restarting;
7651                        restart_schedule = Some(schedule);
7652                        disposition = TerminalDisposition::Restarting;
7653                    } else {
7654                        final_state = ModuleState::Failed;
7655                        disposition = TerminalDisposition::Failed;
7656                        let budget = policy.budget_exhausted_detail();
7657                        disposition_detail =
7658                            Some(disposition_detail.take().map_or_else(
7659                                || budget.clone(),
7660                                |cause| format!("{cause}; {budget}"),
7661                            ));
7662                    }
7663                } else {
7664                    disposition = TerminalDisposition::Disabled;
7665                }
7666            }) {
7667                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7668                return NextAction::Stop {
7669                    registration_released: false,
7670                };
7671            }
7672            if disposition == TerminalDisposition::Failed {
7673                // The window is in the message, not only in the fields: this line
7674                // is read in a scrollback where a bare `max_restarts=3` reads as a
7675                // lifetime cap and sends the operator looking for three crashes
7676                // that never happened together.
7677                error!(
7678                    module_id = %spec.module_id,
7679                    max_restarts = policy.max_restarts,
7680                    window_secs = policy.window.as_secs(),
7681                    "module stopped: {}",
7682                    policy.budget_exhausted_detail()
7683                );
7684            }
7685            if let Err(err) = publish_after_terminal_record(
7686                &spec.module_id,
7687                || {
7688                    record_terminal_with_detail(
7689                        &spec.module_id,
7690                        terminal_ring,
7691                        spawn_events,
7692                        &exit_report,
7693                        disposition,
7694                        disposition_detail,
7695                    )
7696                },
7697                || {
7698                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7699                        state.state = final_state;
7700                        clear_current_process_facts(state);
7701                        state.last_exit = Some(exit_report.clone());
7702                    })
7703                },
7704            ) {
7705                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7706            }
7707
7708            if let Some(schedule) = restart_schedule {
7709                NextAction::Restart {
7710                    schedule: Some(schedule),
7711                }
7712            } else {
7713                let registration_released = match wait_for_registration_release(
7714                    registry,
7715                    &spec.module_id,
7716                    REGISTRY_RELEASE_TIMEOUT,
7717                )
7718                .await
7719                {
7720                    Ok(()) => true,
7721                    Err(err) => {
7722                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7723                        false
7724                    }
7725                };
7726                NextAction::Stop {
7727                    registration_released,
7728                }
7729            }
7730        }
7731        ExitKind::DeliberateSeverance => {
7732            warn!(
7733                module_id = %spec.module_id,
7734                exit_code = ?exit_report.code,
7735                exit_signal = ?exit_report.signal,
7736                "supervised module exited after deliberate connection severance"
7737            );
7738            let should_restart = match lock_snapshot(snapshot) {
7739                Ok(state) => state.enabled,
7740                Err(err) => {
7741                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7742                    return NextAction::Stop {
7743                        registration_released: false,
7744                    };
7745                }
7746            };
7747            let final_state = if should_restart {
7748                ModuleState::Restarting
7749            } else {
7750                ModuleState::Disabled
7751            };
7752            if let Err(err) = publish_after_terminal_record(
7753                &spec.module_id,
7754                || {
7755                    record_terminal(
7756                        &spec.module_id,
7757                        terminal_ring,
7758                        spawn_events,
7759                        &exit_report,
7760                        terminal_disposition(final_state),
7761                    )
7762                },
7763                || {
7764                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7765                        state.state = final_state;
7766                        clear_current_process_facts(state);
7767                        state.last_exit = Some(exit_report.clone());
7768                        state.lifetime_restarts += 1;
7769                    })
7770                },
7771            ) {
7772                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7773                return NextAction::Stop {
7774                    registration_released: false,
7775                };
7776            }
7777
7778            if should_restart {
7779                NextAction::Restart { schedule: None }
7780            } else {
7781                let registration_released = match wait_for_registration_release(
7782                    registry,
7783                    &spec.module_id,
7784                    REGISTRY_RELEASE_TIMEOUT,
7785                )
7786                .await
7787                {
7788                    Ok(()) => true,
7789                    Err(err) => {
7790                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7791                        false
7792                    }
7793                };
7794                NextAction::Stop {
7795                    registration_released,
7796                }
7797            }
7798        }
7799    }
7800}
7801
7802async fn on_child_exit_during_daemon_shutdown(
7803    spec: &ModuleSpec,
7804    registry: &Registry,
7805    snapshot: &SharedSnapshot,
7806    terminal_ring: &Arc<Mutex<TerminalRing>>,
7807    spawn_events: &SpawnEventFeed,
7808    exit_report: ExitReport,
7809) -> NextAction {
7810    info!(
7811        module_id = %spec.module_id,
7812        exit_code = ?exit_report.code,
7813        exit_signal = ?exit_report.signal,
7814        exit_kind = ?exit_report.kind,
7815        "supervised module exited during daemon shutdown; not restarting it"
7816    );
7817    if let Err(err) = publish_after_terminal_record(
7818        &spec.module_id,
7819        || {
7820            record_terminal(
7821                &spec.module_id,
7822                terminal_ring,
7823                spawn_events,
7824                &exit_report,
7825                TerminalDisposition::DaemonShutdown,
7826            )
7827        },
7828        || {
7829            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7830                state.state = ModuleState::Stopped;
7831                clear_current_process_facts(state);
7832                state.last_exit = Some(exit_report.clone());
7833            })
7834        },
7835    ) {
7836        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7837    }
7838    let registration_released =
7839        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7840            .await
7841            .is_ok();
7842    NextAction::Stop {
7843        registration_released,
7844    }
7845}
7846
7847// Keep the status lock out of terminal I/O. Readers may observe the previous
7848// state while recording is in progress, but never a completed exit without its
7849// history. Journal errors are logged by the journal and retained in the ring;
7850// a recording panic still publishes the final state before resuming the unwind.
7851fn publish_after_terminal_record<T>(
7852    module_id: &str,
7853    record: impl FnOnce(),
7854    publish: impl FnOnce() -> T,
7855) -> T {
7856    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7857    let published = publish();
7858    if let Err(panic) = recorded {
7859        error!(
7860            module_id,
7861            "terminal recording panicked; published final state before resuming unwind"
7862        );
7863        std::panic::resume_unwind(panic);
7864    }
7865    published
7866}
7867
7868fn fail_after_wait_error(
7869    module_id: &str,
7870    snapshot: &SharedSnapshot,
7871    terminal_ring: &Arc<Mutex<TerminalRing>>,
7872    spawn_events: &SpawnEventFeed,
7873    detail: String,
7874) {
7875    let report = wait_error_exit_report();
7876    publish_after_terminal_record(
7877        module_id,
7878        || {
7879            record_terminal_with_detail(
7880                module_id,
7881                terminal_ring,
7882                spawn_events,
7883                &report,
7884                TerminalDisposition::Failed,
7885                Some(detail),
7886            )
7887        },
7888        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7889    );
7890}
7891
7892#[cfg(test)]
7893fn record_wait_error_terminal(
7894    module_id: &str,
7895    terminal_ring: &Arc<Mutex<TerminalRing>>,
7896    spawn_events: &SpawnEventFeed,
7897) {
7898    record_terminal(
7899        module_id,
7900        terminal_ring,
7901        spawn_events,
7902        &wait_error_exit_report(),
7903        TerminalDisposition::Failed,
7904    );
7905}
7906
7907fn record_terminal(
7908    module_id: &str,
7909    terminal_ring: &Arc<Mutex<TerminalRing>>,
7910    spawn_events: &SpawnEventFeed,
7911    exit_report: &ExitReport,
7912    disposition: TerminalDisposition,
7913) {
7914    record_terminal_with_detail(
7915        module_id,
7916        terminal_ring,
7917        spawn_events,
7918        exit_report,
7919        disposition,
7920        None,
7921    );
7922}
7923
7924/// The ring lock is held only to capture the read (see
7925/// `TerminalJournal::capture_read`), so this module's exits keep recording
7926/// while the journal files are read. Blocking: it reads files.
7927fn durable_terminal_history_of(
7928    terminal_ring: &Mutex<TerminalRing>,
7929    module_id: &str,
7930) -> subc_control::TerminalHistory {
7931    let read = terminal_ring
7932        .lock()
7933        .unwrap_or_else(|p| p.into_inner())
7934        .capture_durable_history();
7935    read.read(module_id)
7936}
7937
7938fn record_terminal_with_detail(
7939    module_id: &str,
7940    terminal_ring: &Arc<Mutex<TerminalRing>>,
7941    spawn_events: &SpawnEventFeed,
7942    exit_report: &ExitReport,
7943    disposition: TerminalDisposition,
7944    disposition_detail: Option<String>,
7945) {
7946    #[cfg(test)]
7947    terminal_history_tests::before_terminal_record(spawn_events);
7948    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7949    let record = TerminalRecord {
7950        exit_code: exit_report.code,
7951        exit_signal: exit_report.signal,
7952        at_ms: exit_report.at_ms,
7953        disposition,
7954        exit_kind: exit_report.kind.into(),
7955        disposition_detail,
7956    };
7957    terminal_ring
7958        .lock()
7959        .unwrap_or_else(|poisoned| poisoned.into_inner())
7960        .record_exit(module_id, record);
7961}
7962
7963fn untrack_if_registration_released(
7964    process_liveness: &SupervisorProcessLiveness,
7965    registry: &Registry,
7966    module_id: &str,
7967    snapshot: &SharedSnapshot,
7968) {
7969    match registry.get_module(module_id) {
7970        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7971        Ok(Some(_)) => {}
7972        Err(err) => {
7973            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7974        }
7975    }
7976}
7977
7978/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
7979/// then apply the module's configured entries minus daemon-private capture keys.
7980///
7981/// Separated from `spawn_child` only so it can be asserted without spawning a
7982/// process — a duplicate of this logic in a test would pass while the real one
7983/// drifted, which is the defect class this function exists to avoid.
7984/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
7985/// nonce. A `protocol: "none"` module gets neither, because it cannot use
7986/// either and the argument would stop a stock binary from starting at all.
7987/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
7988///
7989/// The plain-spawn form, kept for the tests that assert its plan; spawns go
7990/// through [`apply_wire_spawn_args_for_role`].
7991#[cfg(test)]
7992fn apply_wire_spawn_args(
7993    command: &mut Command,
7994    spec: &ModuleSpec,
7995    connection_file_path: Option<&std::path::Path>,
7996    handle: Option<&SupervisorHandle>,
7997) -> Result<Option<NonceHandoff>, SuperviseError> {
7998    apply_wire_spawn_args_for_role(
7999        command,
8000        spec,
8001        connection_file_path,
8002        handle,
8003        SpawnRole::Plain,
8004    )
8005}
8006
8007/// The read end of a spawn's launch-nonce pipe, prepared by
8008/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8009/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8010/// server before spawn and authenticates its client after the child exists.
8011#[cfg(unix)]
8012type NonceHandoff = subc_os::LaunchNonceHandoff;
8013#[cfg(windows)]
8014type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8015#[cfg(not(any(unix, windows)))]
8016type NonceHandoff = std::convert::Infallible;
8017
8018/// Create the launch nonce that a new module process must present when it
8019/// registers or opens a consumer connection, including a replacement process
8020/// started alongside a still-serving module.
8021///
8022/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8023/// a separate candidate token so the still-serving incumbent and its consumers
8024/// keep their nonce. Both records are installed before the process exists, so
8025/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8026///
8027/// On Unix the nonce is delivered only through a pipe. It is written into
8028/// a pipe whose read end the child gets as descriptor 3, named by
8029/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8030/// process of the same user cannot read it with `ps eww`. That handoff is
8031/// returned rather than installed here, because installing it replaces
8032/// whatever the child has at descriptor 3 and so must be the last pre-exec
8033/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8034/// the Linux callback that writes the process into its cgroup. Only after that
8035/// does `handoff.install_last` register the descriptor-3 replacement, so it
8036/// cannot close a descriptor the cgroup callback still needs.
8037/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8038fn apply_wire_spawn_args_for_role(
8039    command: &mut Command,
8040    spec: &ModuleSpec,
8041    connection_file_path: Option<&std::path::Path>,
8042    handle: Option<&SupervisorHandle>,
8043    role: SpawnRole,
8044) -> Result<Option<NonceHandoff>, SuperviseError> {
8045    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8046    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8047    // included: a daemon started from a module's process tree inherits it,
8048    // and passing it on would point the child at a descriptor it does not
8049    // have.
8050    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8051    #[cfg(windows)]
8052    {
8053        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8054        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8055    }
8056    // Remove inherited or configured copies too: withholding must mean absent.
8057    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8058    if spec.protocol == ModuleProtocol::None {
8059        return Ok(None);
8060    }
8061    if let Some(connection_file_path) = connection_file_path {
8062        command.arg(SUBC_ARG).arg(connection_file_path);
8063    }
8064
8065    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8066    // route.open attestation. Reserved modules additionally use the same nonce
8067    // for HELLO id-squatting protection. A respawn rotates both records.
8068    let nonce = generate_launch_nonce()?;
8069    if let Some(handle) = handle {
8070        match role {
8071            SpawnRole::Plain => {
8072                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8073                if spec.reserved {
8074                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8075                }
8076            }
8077            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8078        }
8079    }
8080    #[cfg(unix)]
8081    let handoff = {
8082        let handoff =
8083            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8084                program: spec.program.clone(),
8085                source,
8086                cgroup_path: None,
8087            })?;
8088        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8089        Some(handoff)
8090    };
8091    #[cfg(windows)]
8092    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8093    #[cfg(not(any(unix, windows)))]
8094    let handoff = None;
8095    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8096    // until the daemon's per-spawn delivery records show that every running
8097    // Windows module has read its secret from the named pipe.
8098    #[cfg(not(unix))]
8099    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8100    Ok(handoff)
8101}
8102
8103fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8104    command.env_remove(CK_LOG_ENV);
8105    // The spawn role is the supervisor's to set, and only on a swap candidate
8106    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8107    // it, is what makes it absent on a plain spawn: the daemon's own
8108    // environment could carry it, and so could a spec built outside daemon
8109    // config (config refuses it as an `env` key). A module reading it on a
8110    // plain restart would pick the long swap budget and leave callers waiting.
8111    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8112    for (key, value) in &spec.env {
8113        // cortexkit-log currently exposes retention only as a Rust struct, not
8114        // environment names. These values are daemon-private sink metadata and
8115        // must never become a public child-process contract by being inherited.
8116        if matches!(
8117            key.as_str(),
8118            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8119        ) || key == SUBC_SPAWN_ROLE_ENV
8120        {
8121            continue;
8122        }
8123        command.env(key, value);
8124    }
8125}
8126
8127#[cfg(windows)]
8128fn prepare_windows_nonce_pipe(
8129    command: &mut Command,
8130    spec: &ModuleSpec,
8131    nonce: &str,
8132) -> Option<NonceHandoff> {
8133    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8134        Ok(handoff) => {
8135            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8136            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8137            // still register until every module reads the pipe. A reader without
8138            // this variable set to env returns the pipe error instead of using
8139            // an inherited or configured environment copy.
8140            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8141            Some(handoff)
8142        }
8143        Err(error) => {
8144            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8145            None
8146        }
8147    }
8148}
8149
8150#[cfg(windows)]
8151fn start_windows_nonce_pipe(
8152    handoff: Option<NonceHandoff>,
8153    pid: u32,
8154    spec: &ModuleSpec,
8155) -> Option<subc_os::LaunchNoncePipeDelivery> {
8156    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8157    // same maximum wait used for registration during a reload. Older modules
8158    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8159    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8160        Ok(delivery) => Some(delivery),
8161        Err(error) => {
8162            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8163            None
8164        }
8165    })
8166}
8167
8168fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8169    #[cfg(windows)]
8170    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8171        return Some(
8172            if snapshot
8173                .pipe_consumed
8174                .as_ref()
8175                .is_some_and(|consumed| consumed.get().is_some())
8176            {
8177                "pipe"
8178            } else {
8179                "env"
8180            }
8181            .to_owned(),
8182        );
8183    }
8184    let _ = snapshot;
8185    None
8186}
8187
8188/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8189/// of a blue/green swap.
8190#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8191enum SpawnRole {
8192    Plain,
8193    SwapCandidate,
8194}
8195
8196/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8197/// `apply_child_env` has already removed the variable for every spawn.
8198fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8199    if role == SpawnRole::SwapCandidate {
8200        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8201    }
8202}
8203
8204fn previous_instance_error(
8205    spec: &ModuleSpec,
8206    pid: u32,
8207    reason: crate::live_children::RetainedReason,
8208) -> SuperviseError {
8209    SuperviseError::Spawn {
8210        program: spec.program.clone(),
8211        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8212        cgroup_path: None,
8213    }
8214}
8215
8216fn spawn_child(
8217    spec: &ModuleSpec,
8218    connection_file_path: Option<&std::path::Path>,
8219    handle: Option<&SupervisorHandle>,
8220    ring: &Arc<Mutex<StderrRing>>,
8221    capture_logs_dir: Option<&std::path::Path>,
8222    roster: &ChildRoster,
8223    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8224) -> Result<SupervisedChild, SuperviseError> {
8225    spawn_child_in_slot(
8226        spec,
8227        connection_file_path,
8228        handle,
8229        ring,
8230        capture_logs_dir,
8231        roster,
8232        #[cfg(target_os = "linux")]
8233        cgroup_placement,
8234        SpawnRole::Plain,
8235        false,
8236    )
8237}
8238
8239/// Spawn one process of `spec` into a slot.
8240///
8241/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8242/// A swap candidate needs a different cgroup from the process it is replacing,
8243/// which is still alive: in the same cgroup the two would be one kill domain,
8244/// and killing a failed candidate could take the incumbent with it.
8245///
8246/// The stderr capture file is `<module_id>.stderr.log` for every process of
8247/// the module, whichever slot it is in, because that is the one file
8248/// `ck module logs` reads. During a swap's overlap both processes append to it;
8249/// the daemon writes whole lines, so the two interleave by line, which is also
8250/// the merged view an operator wants while a swap runs.
8251#[allow(clippy::too_many_arguments)]
8252fn spawn_child_in_slot(
8253    spec: &ModuleSpec,
8254    connection_file_path: Option<&std::path::Path>,
8255    handle: Option<&SupervisorHandle>,
8256    ring: &Arc<Mutex<StderrRing>>,
8257    capture_logs_dir: Option<&std::path::Path>,
8258    roster: &ChildRoster,
8259    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8260    role: SpawnRole,
8261    alternate_slot: bool,
8262) -> Result<SupervisedChild, SuperviseError> {
8263    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8264        return Err(previous_instance_error(spec, pid, reason));
8265    }
8266    if roster.is_closed() {
8267        return Err(SuperviseError::Spawn {
8268            program: spec.program.clone(),
8269            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8270            cgroup_path: None,
8271        });
8272    }
8273    #[cfg(target_os = "linux")]
8274    let cgroup_name = {
8275        // Slot names alone are not kill domains: a retired incumbent may still
8276        // be draining when a later enable/restart spawns into the same slot.
8277        // Decimal entropy keeps the suffix unambiguous; Placement performs
8278        // the module-id escaping and constructs the filesystem path.
8279        if cgroup_placement.is_none() {
8280            swap::cgroup_name(&spec.module_id, alternate_slot)
8281        } else {
8282            let nonce = generate_launch_nonce()?;
8283            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8284            // Leave room for byte escaping and the suffix under NAME_MAX. The
8285            // label is only for humans; the nonce identifies the kill domain.
8286            let mut end = spec.module_id.len().min(64);
8287            while !spec.module_id.is_char_boundary(end) {
8288                end -= 1;
8289            }
8290            format!(
8291                "{}_{suffix}",
8292                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8293            )
8294        }
8295    };
8296    #[cfg(not(target_os = "linux"))]
8297    let _ = alternate_slot;
8298    #[cfg(target_os = "macos")]
8299    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8300    #[cfg(windows)]
8301    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8302        program: spec.program.clone(),
8303        source,
8304        cgroup_path: None,
8305    })?;
8306    #[cfg(windows)]
8307    let mut command = Command::new(image_capture.path());
8308    #[cfg(not(any(target_os = "macos", windows)))]
8309    let mut command = Command::new(&spec.program);
8310    command.args(&spec.args);
8311    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8312    // that is the whole of the intent, so remove that one key rather than the
8313    // environment.
8314    //
8315    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8316    // and took the POSIX environment with it. Modules spawned that way had no
8317    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8318    // logging:
8319    //
8320    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8321    //     both unset it fell back to the temp dir alone and `ck` could not find
8322    //     a daemon running on the same machine from inside any module's process
8323    //     tree — reporting a path the file has never lived at, which reads as
8324    //     "the daemon did not write its file".
8325    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8326    //     the RELATIVE `.local/share`, so a module deriving its own store path
8327    //     resolved it against its own CWD. That is the store-fragmentation
8328    //     defect the daemon already refuses in config (`parse_doc` rejects a
8329    //     relative `storage.data_home`) arriving by derivation instead.
8330    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8331    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8332    //     quietly rather than erroring.
8333    //
8334    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8335    // offered one candidate under /tmp while the file sat in /run/user/1000.
8336    //
8337    // A configured module is unaffected either way: `module_spec()` puts the
8338    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8339    // wins over anything ambient.
8340    apply_child_env(&mut command, spec);
8341    apply_spawn_role(&mut command, role);
8342    let nonce_handoff =
8343        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8344
8345    #[cfg(target_os = "linux")]
8346    let cgroup_path = cgroup_placement
8347        .map(|placement| placement.module_path(&cgroup_name))
8348        .transpose()
8349        .map_err(|source| SuperviseError::Cgroup {
8350            module_id: spec.module_id.clone(),
8351            source,
8352        })?;
8353    #[cfg(not(target_os = "linux"))]
8354    let cgroup_path: Option<PathBuf> = None;
8355    #[cfg(target_os = "linux")]
8356    if let Some(path) = &cgroup_path {
8357        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8358            if let Some(placement) = cgroup_placement {
8359                remove_module_cgroup(placement, &cgroup_name);
8360            }
8361            return Err(error);
8362        }
8363    }
8364
8365    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8366        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8367        match ChildOutputSink::open(&path, capture_retention(spec)) {
8368            Ok(sink) => sink,
8369            Err(error) => {
8370                warn!(
8371                    module_id = %spec.module_id,
8372                    path = %path.display(),
8373                    error = %error,
8374                    "could not open child output capture file; forwarding to stderr"
8375                );
8376                ChildOutputSink::Stderr
8377            }
8378        }
8379    } else {
8380        ChildOutputSink::Stderr
8381    };
8382
8383    command.stdout(Stdio::piped());
8384    command.stderr(Stdio::piped());
8385    command.kill_on_drop(true);
8386    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8387    // before exec). In the daemon's group, a service manager that kills the
8388    // job's process group when the daemon exits (launchd's default) killed
8389    // every module at the same moment its control connection closed, so no
8390    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8391    // module is reached only by the daemon: the EOF it sees when its
8392    // connection closes, and the bounded stop in `child_roster` for anything
8393    // still running after that. On Linux this composes with the cgroup
8394    // placement above: that is a pre_exec write to cgroup.procs, std performs
8395    // setpgid in the child before running pre_exec callbacks, and the two
8396    // change independent process attributes.
8397    //
8398    // stdin is /dev/null because a process outside the terminal's foreground
8399    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8400    // by hand would otherwise hand down. Under a service manager stdin is
8401    // already /dev/null.
8402    #[cfg(unix)]
8403    command.process_group(0);
8404    command.stdin(Stdio::null());
8405    // The LAST pre-exec step, after the cgroup placement above: installing the
8406    // nonce at descriptor 3 replaces whatever the child had there, which could
8407    // be the descriptor an earlier step writes through.
8408    #[cfg(unix)]
8409    if let Some(handoff) = nonce_handoff {
8410        handoff.install_last(command.as_std_mut());
8411    }
8412    #[cfg(not(any(unix, windows)))]
8413    let _ = nonce_handoff;
8414
8415    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8416    // cannot run a single instruction -- and therefore cannot spawn a
8417    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8418    // other two steps and why the window matters.
8419    #[cfg(windows)]
8420    {
8421        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8422            program: spec.program.clone(),
8423            source,
8424            cgroup_path: None,
8425        })?;
8426        subc_jobobject::suspend_on_create_async(&mut command);
8427    }
8428    // Timestamp before the spawn call: on Unix the child can serve before that
8429    // call even returns. This conservative bound never labels an older child
8430    // as a replacement for a request received during process creation.
8431    let spawned_at = Instant::now();
8432    let mut child = match command.spawn() {
8433        Ok(child) => child,
8434        Err(source) => {
8435            #[cfg(target_os = "linux")]
8436            if let Some(placement) = cgroup_placement {
8437                remove_module_cgroup(placement, &cgroup_name);
8438            }
8439            return Err(SuperviseError::Spawn {
8440                program: spec.program.clone(),
8441                source,
8442                cgroup_path,
8443            });
8444        }
8445    };
8446    #[cfg(test)]
8447    spawn_record_tests::after_create(&child, spec);
8448    // The parent must close its writer now: the acknowledgement pipe reports EOF
8449    // only when every writer is gone, and the child's copy closes when the
8450    // trampoline replaces itself with the module. Command holds only an integer
8451    // in its pre_exec callback, not another writer.
8452    #[cfg(target_os = "macos")]
8453    drop(exec_ack);
8454
8455    // Bind while the child is still suspended, before job assignment resumes it.
8456    // Keeping the executable handle open across CreateProcessW denies rename
8457    // and deletion, so the file cannot be exchanged during spawn. Bind that
8458    // file ID to the process handle returned by spawn, never a reopened PID.
8459    #[cfg(windows)]
8460    let windows_image = match image_capture.bind_tokio(&child) {
8461        Ok(image) => Arc::new(image),
8462        Err(source) => {
8463            kill_and_reap_rejected_child(&mut child);
8464            return Err(SuperviseError::Spawn {
8465                program: spec.program.clone(),
8466                source,
8467                cgroup_path: None,
8468            });
8469        }
8470    };
8471    // Assign the suspended child to its module job; resume only after its
8472    // process identity has been durably recorded for orphan recovery.
8473    #[cfg(windows)]
8474    let job = contain_spawned_child(&mut child, spec)?;
8475    let spawned_at_ms = unix_ms_now();
8476    #[cfg(not(windows))]
8477    let spawned_from = spec.program.clone();
8478    #[cfg(windows)]
8479    let spawned_from = windows_image.path().to_path_buf();
8480    #[cfg(not(windows))]
8481    let spawned_file_identity = spawned_file_identity(&spawned_from);
8482    #[cfg(windows)]
8483    let spawned_file_identity = Some(SpawnedFileIdentity {
8484        device: 0,
8485        inode: 0,
8486        windows_image: Some(Arc::clone(&windows_image)),
8487    });
8488    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8489        program: spec.program.clone(),
8490        source: io::Error::other("spawned child exposed no live pid"),
8491        cgroup_path: cgroup_path.clone(),
8492    })?;
8493    #[cfg(windows)]
8494    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8495    #[cfg(not(windows))]
8496    let process_start_time = crate::provenance::process_start_time(pid);
8497    #[cfg(windows)]
8498    let process_start_time = Some(windows_image.start_time());
8499    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8500    #[cfg(all(test, target_os = "macos"))]
8501    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8502    // Record the committed kernel image, not the configured pathname: PATH
8503    // lookup and shebang interpretation can select a different executable.
8504    // Linux's posix_spawn can return before that image is installed, so the
8505    // observer also waits for the kernel's first-exec flag to clear.
8506    #[cfg(not(windows))]
8507    let recorded_image = observe_spawned_image(pid);
8508    // spawn() confirms only the first exec, into the trampoline. Never persist
8509    // the trampoline image; the asynchronous acknowledgement publishes the
8510    // module image once the trampoline has replaced itself with the module.
8511    #[cfg(target_os = "macos")]
8512    let recorded_image = if privacy_exec.is_some() {
8513        None
8514    } else {
8515        recorded_image
8516    };
8517    #[cfg(target_os = "linux")]
8518    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8519    #[cfg(not(target_os = "linux"))]
8520    let recorded_cgroup_name = None;
8521    let roster_guard = roster.admit(
8522        spec.module_id.clone(),
8523        pid,
8524        spec.protocol,
8525        process_start_time,
8526        crate::child_roster::RecordedIdentity {
8527            #[cfg(not(windows))]
8528            start_time: recorded_image.map(|image| image.start_time),
8529            #[cfg(windows)]
8530            start_time: process_start_time,
8531            #[cfg(not(windows))]
8532            executable: recorded_image
8533                .and_then(|image| image.executable)
8534                .map(crate::live_children::ExecutableIdentity::from),
8535            #[cfg(windows)]
8536            executable: Some(crate::live_children::ExecutableIdentity::from(
8537                windows_image.identity(),
8538            )),
8539            cgroup_name: recorded_cgroup_name,
8540            #[cfg(target_os = "linux")]
8541            cgroup_placement: cgroup_placement.cloned(),
8542        },
8543    );
8544    let roster_guard = match roster_guard {
8545        Ok(guard) => guard,
8546        Err(source) => {
8547            #[cfg(windows)]
8548            if let Some(job) = &job {
8549                let _ = job.terminate();
8550            }
8551            #[cfg(target_os = "linux")]
8552            kill_module_cgroup(cgroup_placement, &cgroup_name);
8553            kill_and_reap_rejected_child(&mut child);
8554            #[cfg(target_os = "linux")]
8555            if let Some(placement) = cgroup_placement {
8556                remove_module_cgroup(placement, &cgroup_name);
8557            }
8558            return Err(SuperviseError::Spawn {
8559                program: spec.program.clone(),
8560                source: io::Error::other(format!(
8561                    "could not durably record module child: {source}"
8562                )),
8563                cgroup_path,
8564            });
8565        }
8566    };
8567    // The check at the top of this function can pass just before daemon
8568    // shutdown begins, and the process is only in the roster from here on.
8569    // The shutdown stop returns as soon as it finds the roster empty, so a
8570    // process admitted after that look would outlive the daemon. The roster
8571    // is closed before the stop first reads it and admission happens under
8572    // the roster's lock, so either the stop sees this process or this check
8573    // sees the roster closed: end the process now rather than start a module
8574    // the daemon is about to stop.
8575    if roster.is_closed() {
8576        // This child was never admitted, so there is no module protocol shutdown to wait for.
8577        #[cfg(target_os = "linux")]
8578        kill_module_cgroup(cgroup_placement, &cgroup_name);
8579        if let Err(error) = child.start_kill() {
8580            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8581        }
8582        #[cfg(target_os = "linux")]
8583        if let Some(placement) = cgroup_placement {
8584            // This spawn was never admitted, so shutdown has no roster entry
8585            // to await. Do not detach its cleanup: the runtime could exit
8586            // before that task reaps the rejected child and removes its group.
8587            while matches!(child.try_wait(), Ok(None)) {
8588                std::thread::yield_now();
8589            }
8590            if matches!(
8591                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8592                subc_cgroup::KillOutcome::Killed
8593            ) {
8594                if let Ok(path) = placement.module_path(&cgroup_name) {
8595                    while std::fs::read_to_string(path.join("cgroup.events"))
8596                        .ok()
8597                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8598                    {
8599                        std::thread::yield_now();
8600                    }
8601                }
8602            }
8603            remove_module_cgroup(placement, &cgroup_name);
8604        }
8605        drop(roster_guard);
8606        return Err(SuperviseError::Spawn {
8607            program: spec.program.clone(),
8608            source: io::Error::other(
8609                "the daemon began shutting down while this process was starting; ended it",
8610            ),
8611            cgroup_path,
8612        });
8613    }
8614
8615    #[cfg(windows)]
8616    if let Err(error) = resume_suspended_child(pid, spec) {
8617        kill_and_reap_rejected_child(&mut child);
8618        return Err(error);
8619    }
8620
8621    let stdout_pump = match child.stdout.take() {
8622        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8623        None => {
8624            warn!(
8625                module_id = %spec.module_id,
8626                "spawned child exposed no stdout pipe; file capture will be incomplete"
8627            );
8628            None
8629        }
8630    };
8631    let stderr_pump = match child.stderr.take() {
8632        Some(stderr) => {
8633            let generation = ring
8634                .lock()
8635                .unwrap_or_else(|poisoned| poisoned.into_inner())
8636                .begin_process();
8637            Some(StderrPump {
8638                task: tokio::spawn(pump_stderr_to(
8639                    stderr,
8640                    Arc::clone(ring),
8641                    generation,
8642                    output_sink,
8643                )),
8644                generation,
8645            })
8646        }
8647        None => {
8648            // Spawning succeeded but the pipe did not materialise. Recording it as
8649            // uncaptured keeps the tail honest: the alternative is an empty tail
8650            // that reads as a module which printed nothing.
8651            ring.lock()
8652                .unwrap_or_else(|poisoned| poisoned.into_inner())
8653                .mark_not_captured("stderr pipe was not available on spawn");
8654            warn!(
8655                module_id = %spec.module_id,
8656                "spawned child exposed no stderr pipe; tail will be unavailable"
8657            );
8658            None
8659        }
8660    };
8661
8662    let exec = Arc::new(crate::activity::ExecOutcome::new());
8663    #[cfg(target_os = "macos")]
8664    let pending_exec = privacy_exec.is_some();
8665    #[cfg(not(target_os = "macos"))]
8666    let pending_exec = false;
8667    if !pending_exec {
8668        exec.finish(crate::activity::ExecState::CommittedExec);
8669    }
8670    Ok(SupervisedChild {
8671        child,
8672        exec: crate::activity::ExecOwner(exec),
8673        #[cfg(windows)]
8674        nonce_delivery,
8675        protocol: spec.protocol,
8676        #[cfg(target_os = "linux")]
8677        module_id: cgroup_name,
8678        #[cfg(target_os = "linux")]
8679        cgroup_placement: cgroup_placement.cloned(),
8680        #[cfg(windows)]
8681        job,
8682        stdout_pump,
8683        stderr_pump,
8684        stderr_ring: Arc::clone(ring),
8685        spawned_at,
8686        spawned_at_ms,
8687        spawned_from,
8688        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8689        spawned_file_identity,
8690        process_start_time,
8691        process_identity,
8692        pid,
8693        roster_guard: Some(roster_guard),
8694        #[cfg(target_os = "macos")]
8695        privacy_exec,
8696        #[cfg(target_os = "macos")]
8697        report_ready: Arc::new(OnceLock::new()),
8698        spawn_failure: None,
8699    })
8700}
8701
8702#[cfg(test)]
8703mod spawn_record_tests {
8704    use super::*;
8705
8706    thread_local! {
8707        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8708    }
8709
8710    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8711        LAST_PID.set(child.id());
8712        #[cfg(windows)]
8713        if let Some((_, address)) = _spec
8714            .env
8715            .iter()
8716            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8717        {
8718            use std::io::{Read, Write};
8719            let mut stream = std::net::TcpStream::connect(address).unwrap();
8720            stream
8721                .write_all(&child.id().unwrap().to_ne_bytes())
8722                .unwrap();
8723            let mut release = [0];
8724            stream.read_exact(&mut release).unwrap();
8725        }
8726    }
8727
8728    #[cfg(windows)]
8729    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8730        if spec
8731            .env
8732            .iter()
8733            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8734        {
8735            Err(io::Error::other("injected module job assignment failure"))
8736        } else {
8737            Ok(())
8738        }
8739    }
8740
8741    fn spec() -> ModuleSpec {
8742        ModuleSpec {
8743            module_id: "spawn-record-worker".into(),
8744            #[cfg(unix)]
8745            program: "/bin/sleep".into(),
8746            #[cfg(windows)]
8747            program: std::env::current_exe().unwrap(),
8748            #[cfg(unix)]
8749            args: vec!["60".into()],
8750            #[cfg(windows)]
8751            args: vec![
8752                "--exact".into(),
8753                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8754                "--nocapture".into(),
8755            ],
8756            env: vec![],
8757            reserved: false,
8758            reserved_prefixes: vec![],
8759            protocol: ModuleProtocol::None,
8760            overlap: Default::default(),
8761        }
8762    }
8763
8764    #[tokio::test]
8765    async fn record_sync_failure_fails_spawn_and_reaps_child() {
8766        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8767        let supervisor =
8768            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8769                .with_live_children_record(home.join("live-children.json"));
8770        crate::live_children::fail_next_file_sync();
8771        let result = supervisor.spawn(spec());
8772        let failed = result.is_err();
8773        if let Ok(module) = result {
8774            module.stop().await.unwrap();
8775        }
8776        assert!(failed, "record failure must fail spawn");
8777        let pid = LAST_PID.get().expect("the real OS child was created");
8778        assert!(
8779            subc_os::Process::open(pid).unwrap().is_none(),
8780            "child {pid} survived rejected admission or was not reaped"
8781        );
8782    }
8783
8784    #[cfg(target_os = "macos")]
8785    #[test]
8786    #[ignore = "manual on-device latency measurement"]
8787    fn durable_record_write_latency_50() {
8788        let home = cortexkit_test_support::ScratchDir::new("record-latency");
8789        let path = home.join("live-children.json");
8790        let children = vec![crate::live_children::LiveChild {
8791            module_id: "latency-worker".into(),
8792            pid: std::process::id(),
8793            protocol: ModuleProtocol::None,
8794            start_time: Some(1),
8795            executable: None,
8796            cgroup_name: None,
8797        }];
8798        let mut samples = Vec::new();
8799        for _ in 0..50 {
8800            let start = std::time::Instant::now();
8801            crate::live_children::write_record(&path, &children).unwrap();
8802            samples.push(start.elapsed());
8803        }
8804        samples.sort();
8805        eprintln!(
8806            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8807            (samples[24] + samples[25]) / 2,
8808            samples[47]
8809        );
8810    }
8811
8812    #[cfg(windows)]
8813    #[test]
8814    fn windows_store_touch_fixture() {
8815        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8816            std::fs::write(path, b"module ran").unwrap();
8817        }
8818    }
8819
8820    #[cfg(windows)]
8821    #[tokio::test]
8822    async fn windows_daemon_fixture() {
8823        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8824            return;
8825        };
8826        let mut spec = spec();
8827        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8828        let supervisor =
8829            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8830        let _module = supervisor.spawn(spec).unwrap();
8831    }
8832
8833    #[cfg(windows)]
8834    #[test]
8835    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8836        use std::io::Read;
8837        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8838        // An outer job models a scheduler/service host. The daemon's root and
8839        // root job must nest without requiring breakaway from this job.
8840        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8841        command
8842            .args([
8843                "--exact",
8844                "supervise::spawn_record_tests::windows_daemon_fixture",
8845                "--nocapture",
8846            ])
8847            .env(
8848                "SUBC_TEST_DAEMON_BARRIER",
8849                listener.local_addr().unwrap().to_string(),
8850            );
8851        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8852        listener.set_nonblocking(true).unwrap();
8853        let deadline = std::time::Instant::now() + Duration::from_secs(30);
8854        let mut stream = loop {
8855            if let Ok((stream, _)) = listener.accept() {
8856                break stream;
8857            }
8858            assert!(
8859                daemon.child.try_wait().unwrap().is_none(),
8860                "daemon failed before rendezvous"
8861            );
8862            assert!(
8863                std::time::Instant::now() < deadline,
8864                "daemon did not reach CreateProcess failpoint"
8865            );
8866            std::thread::yield_now();
8867        };
8868        stream
8869            .set_read_timeout(Some(Duration::from_secs(30)))
8870            .unwrap();
8871        let mut pid = [0; 4];
8872        stream.read_exact(&mut pid).unwrap();
8873        let pid = u32::from_ne_bytes(pid);
8874        assert!(subc_jobobject::process_exists(pid));
8875        // Terminate only the daemon, not its outer job: otherwise the host job
8876        // would also kill a child whose daemon never joined its root job.
8877        daemon.child.kill().unwrap();
8878        daemon.child.wait().unwrap();
8879        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8880        if !exited {
8881            daemon.job.terminate().unwrap();
8882        }
8883        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8884    }
8885
8886    #[cfg(windows)]
8887    #[tokio::test]
8888    async fn windows_containment_failure_never_resumes() {
8889        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8890        let sentinel = home.join("store-touched");
8891        let mut spec = spec();
8892        spec.env
8893            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8894        spec.env.push((
8895            "SUBC_TEST_STORE_TOUCH".into(),
8896            sentinel.display().to_string(),
8897        ));
8898        let supervisor =
8899            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8900        let result = supervisor.spawn(spec);
8901        // A version that resumes despite assignment failure must have time to
8902        // execute its sentinel before dropping the returned module kills it.
8903        if let Ok(module) = result {
8904            let _ = tokio::time::timeout(Duration::from_secs(10), async {
8905                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8906                    tokio::task::yield_now().await;
8907                }
8908            })
8909            .await;
8910            let _ = module.stop().await;
8911            panic!(
8912                "assignment failure resumed a module; store touched={}",
8913                sentinel.exists()
8914            );
8915        }
8916        assert!(
8917            !sentinel.exists(),
8918            "refused module executed store-touch sentinel"
8919        );
8920        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8921    }
8922
8923    #[cfg(windows)]
8924    #[tokio::test]
8925    async fn windows_nested_daemon_fixture() {
8926        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8927            return;
8928        };
8929        let mut spec = spec();
8930        spec.env.push((
8931            "SUBC_TEST_STORE_TOUCH".into(),
8932            sentinel.to_str().unwrap().into(),
8933        ));
8934        let supervisor =
8935            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8936        let module = supervisor
8937            .spawn(spec)
8938            .expect("per-module job must nest below daemon and host jobs");
8939        tokio::time::timeout(Duration::from_secs(10), async {
8940            while !PathBuf::from(&sentinel).exists() {
8941                tokio::task::yield_now().await;
8942            }
8943        })
8944        .await
8945        .unwrap();
8946        module.stop().await.unwrap();
8947    }
8948
8949    #[cfg(windows)]
8950    #[test]
8951    fn windows_module_job_nests_under_host_job() {
8952        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8953        let sentinel = home.join("nested-store-touched");
8954        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8955        command
8956            .args([
8957                "--exact",
8958                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8959                "--nocapture",
8960            ])
8961            .env("SUBC_TEST_NESTED_STORE", &sentinel);
8962        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8963        assert!(daemon.child.wait().unwrap().success());
8964        assert!(sentinel.exists(), "nested protocol:none module never ran");
8965    }
8966}
8967
8968#[cfg(target_os = "linux")]
8969pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8970    use subc_cgroup::KillOutcome;
8971    match subc_cgroup::kill_module(placement, module_id) {
8972        KillOutcome::Killed => {}
8973        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8974            debug!(
8975                module_id,
8976                "cgroup tree kill unavailable; using direct-child kill"
8977            );
8978        }
8979        KillOutcome::IoError { path, error } => {
8980            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8981        }
8982    }
8983}
8984
8985/// Assign a freshly spawned Windows child without ever resuming on failure.
8986///
8987/// Steps 2 and 3 of the suspended-create contract: the job is created and the
8988/// child assigned **while it is still suspended** (step 1 is
8989/// `suspend_on_create_async` at the spawn site). The caller resumes only after
8990/// the child's roster admission is durable. Failed containment kills and reaps
8991/// the suspended child instead of running an unprotected module.
8992#[cfg(windows)]
8993fn contain_spawned_child(
8994    child: &mut Child,
8995    spec: &ModuleSpec,
8996) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8997    let assignment = || {
8998        let job = subc_jobobject::JobObject::new()?;
8999        #[cfg(test)]
9000        spawn_record_tests::assignment_failpoint(spec)?;
9001        job.assign(child)?;
9002        Ok::<_, io::Error>(job)
9003    };
9004    match assignment() {
9005        Ok(job) => Ok(Some(job)),
9006        Err(source) => {
9007            kill_and_reap_rejected_child(child);
9008            Err(SuperviseError::Spawn {
9009                program: spec.program.clone(),
9010                source: io::Error::other(format!(
9011                    "Windows module job containment failed: {source}"
9012                )),
9013                cgroup_path: None,
9014            })
9015        }
9016    }
9017}
9018
9019/// Rejection is synchronous: do not return a spawn error while the child can
9020/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9021fn kill_and_reap_rejected_child(child: &mut Child) {
9022    #[cfg(unix)]
9023    if let Some(pid) = child
9024        .id()
9025        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9026    {
9027        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9028    }
9029    let _ = child.start_kill();
9030    while matches!(child.try_wait(), Ok(None)) {
9031        std::thread::yield_now();
9032    }
9033}
9034
9035/// Resume a suspended child, killing it if it cannot be started.
9036///
9037/// A suspended process holds a pid and does nothing, so there is no useful
9038/// state to return: the caller gets an error and the spawn fails.
9039#[cfg(windows)]
9040fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9041    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9042        // Kill it here rather than leaving a suspended process for the caller
9043        // to notice; `kill_on_drop` would eventually do this, but the module
9044        // would have been reported as running in between.
9045        let _ = std::process::Command::new("taskkill.exe")
9046            .args(["/PID", &pid.to_string(), "/T", "/F"])
9047            .stdin(Stdio::null())
9048            .stdout(Stdio::null())
9049            .stderr(Stdio::null())
9050            .status();
9051        return Err(SuperviseError::Spawn {
9052            program: spec.program.clone(),
9053            source,
9054            cgroup_path: None,
9055        });
9056    }
9057    Ok(())
9058}
9059
9060#[cfg(target_os = "linux")]
9061fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9062    match placement.remove_module(module_id) {
9063        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9064        Err(error) => warn!(
9065            module_id,
9066            error = %error,
9067            "could not remove module cgroup after process exit; continuing teardown"
9068        ),
9069    }
9070}
9071
9072#[cfg(target_os = "linux")]
9073async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9074    // Reaping the direct child is not proof its descendants exited. End the
9075    // residual tree and wait for the kernel's population fact before rmdir;
9076    // otherwise a successful parent wait leaks a directory on each restart.
9077    if matches!(
9078        subc_cgroup::kill_module(Some(placement), module_id),
9079        subc_cgroup::KillOutcome::Killed
9080    ) {
9081        if let Ok(path) = placement.module_path(module_id) {
9082            while std::fs::read_to_string(path.join("cgroup.events"))
9083                .ok()
9084                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9085            {
9086                sleep(Duration::from_millis(1)).await;
9087            }
9088        }
9089    }
9090    remove_module_cgroup(placement, module_id);
9091}
9092
9093#[cfg(target_os = "linux")]
9094fn apply_cgroup_placement(
9095    command: &mut Command,
9096    spec: &ModuleSpec,
9097    path: &std::path::Path,
9098) -> Result<(), SuperviseError> {
9099    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9100        module_id: spec.module_id.clone(),
9101        source,
9102    })
9103}
9104
9105fn capture_retention(spec: &ModuleSpec) -> Retention {
9106    let defaults = Retention::default();
9107    let value = |name: &str| {
9108        spec.env
9109            .iter()
9110            .rev()
9111            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9112    };
9113    Retention {
9114        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9115            .and_then(|value| value.parse().ok())
9116            .unwrap_or(defaults.max_file_mb),
9117        keep: value(CAPTURE_KEEP_ENV)
9118            .and_then(|value| value.parse().ok())
9119            .unwrap_or(defaults.keep),
9120        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9121            .and_then(|value| value.parse().ok())
9122            .unwrap_or(defaults.max_age_days),
9123    }
9124}
9125
9126/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9127/// module's registration to the exact process the supervisor spawned.
9128fn generate_launch_nonce() -> Result<String, SuperviseError> {
9129    let mut bytes = [0u8; 32];
9130    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9131        reason: source.to_string(),
9132    })?;
9133    let mut hex = String::with_capacity(64);
9134    for b in bytes {
9135        use std::fmt::Write;
9136        let _ = write!(hex, "{b:02x}");
9137    }
9138    Ok(hex)
9139}
9140
9141/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9142/// signal about how many leading bytes matched.
9143fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9144    if a.len() != b.len() {
9145        return false;
9146    }
9147    let mut diff = 0u8;
9148    for (x, y) in a.iter().zip(b.iter()) {
9149        diff |= x ^ y;
9150    }
9151    diff == 0
9152}
9153
9154/// The kernel's image after an acknowledged exec, shared by ordinary launches
9155/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9156/// not identities inferred from a configured pathname.
9157#[cfg(not(windows))]
9158fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9159    let process = subc_os::Process::open(pid).ok().flatten()?;
9160    #[cfg(target_os = "linux")]
9161    return sample_after_linux_exec(
9162        || {
9163            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9164            let committed = linux_exec_committed(&stat);
9165            #[cfg(test)]
9166            spawn_image_tests::report_exec_state(committed);
9167            committed
9168        },
9169        || process.observe(),
9170    );
9171    #[cfg(not(target_os = "linux"))]
9172    process.observe()
9173}
9174
9175/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9176/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9177/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9178#[cfg(target_os = "linux")]
9179fn linux_exec_committed(stat: &str) -> Option<bool> {
9180    const PF_FORKNOEXEC: u64 = 0x40;
9181    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9182    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9183}
9184
9185#[cfg(target_os = "linux")]
9186fn sample_after_linux_exec(
9187    mut exec_committed: impl FnMut() -> Option<bool>,
9188    observe: impl FnOnce() -> Option<subc_os::Observation>,
9189) -> Option<subc_os::Observation> {
9190    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9191    while !exec_committed()? {
9192        // A failed or indefinitely stalled exec must not publish the inherited
9193        // image as authority to signal a future orphan. Missing identity keeps
9194        // the existing conservative orphan-sweep behavior.
9195        if std::time::Instant::now() >= deadline {
9196            return None;
9197        }
9198        std::thread::sleep(Duration::from_micros(50));
9199    }
9200    observe()
9201}
9202
9203#[cfg(all(test, target_os = "linux"))]
9204mod spawn_image_tests {
9205    use super::*;
9206    use std::cell::{Cell, RefCell};
9207
9208    enum ExecEvent {
9209        Uncommitted,
9210        Image(Option<subc_os::Observation>),
9211    }
9212
9213    thread_local! {
9214        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9215    }
9216
9217    pub(super) fn report_exec_state(committed: Option<bool>) {
9218        if committed == Some(false) {
9219            EXEC_EVENTS.with(|events| {
9220                if let Some(events) = events.borrow_mut().take() {
9221                    let _ = events.send(ExecEvent::Uncommitted);
9222                }
9223            });
9224        }
9225    }
9226
9227    #[test]
9228    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9229        use std::io::{Read, Write};
9230        use std::os::fd::AsRawFd;
9231
9232        let descriptor = std::fs::File::open("/dev/null").unwrap();
9233        let mut command = std::process::Command::new("/bin/sleep");
9234        command.arg("30");
9235        let (mut ready, mut release) =
9236            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9237                .unwrap();
9238        let spawn = std::thread::spawn(move || command.spawn());
9239        let mut message = [0u8; 8];
9240        ready.read_exact(&mut message).unwrap();
9241        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9242        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9243        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9244        let (events, received) = std::sync::mpsc::channel();
9245        let observer = std::thread::spawn(move || {
9246            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9247            let image = observe_spawned_image(pid);
9248            let _ = events.send(ExecEvent::Image(image));
9249        });
9250        // Release only once the production observer has seen the uncommitted
9251        // exec, or has incorrectly returned the inherited image. Either way,
9252        // release and reap before asserting so a regression cannot leak a child.
9253        let first = received.recv_timeout(Duration::from_secs(5));
9254        release.write_all(&[1]).unwrap();
9255        let mut child = spawn.join().unwrap().unwrap();
9256        let after = match first {
9257            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9258                Ok(ExecEvent::Image(image)) => image,
9259                _ => None,
9260            },
9261            Ok(ExecEvent::Image(image)) => image,
9262            Err(_) => None,
9263        };
9264        observer.join().unwrap();
9265        child.kill().unwrap();
9266        child.wait().unwrap();
9267
9268        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9269        assert_eq!(
9270            inherited.unwrap().executable,
9271            subc_os::file_identity(&std::env::current_exe().unwrap())
9272        );
9273        assert_eq!(
9274            after.unwrap().executable,
9275            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9276        );
9277    }
9278
9279    #[test]
9280    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9281        let committed = Cell::new(false);
9282        let inherited = subc_os::Observation {
9283            start_time: 23,
9284            executable: Some(subc_os::FileIdentity {
9285                device: 29,
9286                inode: 31,
9287            }),
9288        };
9289        let module = subc_os::Observation {
9290            start_time: 23,
9291            executable: Some(subc_os::FileIdentity {
9292                device: 29,
9293                inode: 37,
9294            }),
9295        };
9296        let polls = Cell::new(0);
9297        let image = sample_after_linux_exec(
9298            || {
9299                polls.set(polls.get() + 1);
9300                // Hold the inherited image through the first poll, as in the
9301                // posix_spawn parent-wake window, then commit the module image.
9302                committed.set(polls.get() == 2);
9303                Some(committed.get())
9304            },
9305            || Some(if committed.get() { module } else { inherited }),
9306        );
9307        assert_eq!(image, Some(module));
9308        assert_eq!(polls.get(), 2);
9309    }
9310
9311    #[test]
9312    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9313        assert_eq!(
9314            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9315            Some(false)
9316        );
9317        assert_eq!(
9318            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9319            Some(true)
9320        );
9321        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9322        assert_eq!(linux_exec_committed("17 (a) R"), None);
9323    }
9324
9325    #[test]
9326    fn unreadable_exec_state_never_samples_an_inherited_image() {
9327        assert_eq!(
9328            sample_after_linux_exec(
9329                || None,
9330                || panic!("must not sample before exec confirmation")
9331            ),
9332            None
9333        );
9334    }
9335
9336    #[test]
9337    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9338        assert_eq!(
9339            sample_after_linux_exec(
9340                || Some(false),
9341                || panic!("must not sample after an exec timeout")
9342            ),
9343            None
9344        );
9345    }
9346}
9347
9348/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9349/// different executable after the no-delete capture was taken.
9350#[cfg(windows)]
9351fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9352    let mut program = spec.program.clone();
9353    if program.is_absolute() || program.components().count() > 1 {
9354        // An explicit path can name a PE image with no extension. Try that
9355        // exact file first, as Command does, before falling back to .exe.
9356        match subc_os::ExecutableCapture::open(&program) {
9357            Ok(capture) => return Ok(capture),
9358            Err(error)
9359                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9360            Err(error) => return Err(error),
9361        }
9362        program.set_extension("exe");
9363        return subc_os::ExecutableCapture::open(&program);
9364    }
9365    if program.extension().is_none() {
9366        program.set_extension("exe");
9367    }
9368    let custom_path = spec
9369        .env
9370        .iter()
9371        .rev()
9372        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9373    let mut directories = Vec::new();
9374    // These are Windows' default executable search locations. A configured
9375    // PATH still overrides the inherited PATH, not the system directories.
9376    if let Some(parent) = std::env::current_exe()?.parent() {
9377        directories.push(parent.to_path_buf());
9378    }
9379    directories.push(std::env::current_dir()?);
9380    if let Some(windows) = std::env::var_os("SystemRoot") {
9381        let windows = PathBuf::from(windows);
9382        directories.push(windows.join("System32"));
9383        directories.push(windows.clone());
9384    }
9385    let path = custom_path
9386        .map(|(_, value)| std::ffi::OsString::from(value))
9387        .or_else(|| std::env::var_os("PATH"));
9388    if let Some(path) = path {
9389        directories.extend(std::env::split_paths(&path));
9390    }
9391    for directory in directories {
9392        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9393            Ok(capture) => return Ok(capture),
9394            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9395            Err(error) => return Err(error),
9396        }
9397    }
9398    Err(io::Error::new(
9399        io::ErrorKind::NotFound,
9400        "resolved executable not found",
9401    ))
9402}
9403
9404fn spawn_and_mark_running(
9405    spec: &ModuleSpec,
9406    runtime: &SupervisorRuntimeConfig,
9407    snapshot: &SharedSnapshot,
9408) -> Result<SupervisedChild, SuperviseError> {
9409    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9410    let child = spawn_child(
9411        spec,
9412        runtime.connection_file_path.as_deref(),
9413        runtime.supervisor_handle.as_ref(),
9414        &runtime.stderr_ring,
9415        runtime.capture_logs_dir.as_deref(),
9416        &runtime.child_roster,
9417        #[cfg(target_os = "linux")]
9418        runtime.cgroup_placement.as_ref(),
9419    )?;
9420    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9421    if let Some(spawn) = activity_spawn {
9422        spawn.complete(snapshot);
9423    }
9424    Ok(child)
9425}
9426
9427enum RegistrationWaitOutcome {
9428    Registered,
9429    Exited(ExitReport),
9430    TimedOut,
9431}
9432
9433struct ReloadRegistrationFailure {
9434    exit_report: ExitReport,
9435    reason: String,
9436}
9437
9438#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9439enum BusyGaugeObservation {
9440    Quiescent,
9441    Busy,
9442    Omitted,
9443}
9444
9445fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9446    let Some(metrics) = metrics.and_then(Value::as_object) else {
9447        return BusyGaugeObservation::Omitted;
9448    };
9449    let mut sum = 0u128;
9450    for gauge in gauges {
9451        let Some(value) = metrics.get(gauge) else {
9452            return BusyGaugeObservation::Omitted;
9453        };
9454        let Some(value) = value.as_u64() else {
9455            return BusyGaugeObservation::Busy;
9456        };
9457        sum = sum.saturating_add(u128::from(value));
9458    }
9459    if sum == 0 {
9460        BusyGaugeObservation::Quiescent
9461    } else {
9462        BusyGaugeObservation::Busy
9463    }
9464}
9465
9466fn declared_busy_gauges(
9467    registry: &Registry,
9468    module_id: &str,
9469) -> Result<Vec<String>, SuperviseError> {
9470    busy_gauges_of(
9471        registry
9472            .get_module(module_id)
9473            .map_err(SuperviseError::Registry)?,
9474    )
9475}
9476
9477/// [`declared_busy_gauges`] for the registration a connection holds, in any
9478/// slot: after cutover the incumbent is no longer the id's active
9479/// registration, and its own manifest is the one that names its gauges.
9480fn declared_busy_gauges_for_connection(
9481    registry: &Registry,
9482    connection_id: ConnectionId,
9483) -> Result<Vec<String>, SuperviseError> {
9484    busy_gauges_of(
9485        registry
9486            .get_module_by_connection(connection_id)
9487            .map_err(SuperviseError::Registry)?,
9488    )
9489}
9490
9491fn busy_gauges_of(
9492    registration: Option<crate::registry::ModuleRegistration>,
9493) -> Result<Vec<String>, SuperviseError> {
9494    let Some(registration) = registration else {
9495        return Ok(Vec::new());
9496    };
9497    let Some(self_signals) = registration.manifest.self_signals else {
9498        return Ok(Vec::new());
9499    };
9500
9501    let mut gauges = Vec::new();
9502    for declaration in self_signals {
9503        if declaration.kind != SelfSignalKind::Busy {
9504            continue;
9505        }
9506        match declaration.anchored_to {
9507            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9508                gauges.extend(declared)
9509            }
9510            _ => {
9511                // An invalid Busy anchor is fail-safe: the empty name cannot be
9512                // present in a conforming health report, so this drain stays busy.
9513                gauges.push(String::new());
9514            }
9515        }
9516    }
9517    Ok(gauges)
9518}
9519
9520/// Wait for `endpoint` to have nothing in flight and, when the module declares
9521/// busy gauges, for a health probe to report them quiet. The probe is addressed
9522/// by `scope`: a swap's superseded incumbent must be asked about its own
9523/// gauges, and by module id the probe would reach the promoted candidate.
9524/// While the gauges read busy, the probe repeats every
9525/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9526async fn wait_for_forwarding_quiescence(
9527    forwarding: &ForwardingTable,
9528    module_id: &str,
9529    runtime: &SupervisorRuntimeConfig,
9530    endpoint: crate::ModuleEndpointId,
9531    deadline: Instant,
9532    busy_gauges: &[String],
9533    scope: DrainScope,
9534) -> Result<bool, SuperviseError> {
9535    let mut gauges_quiescent = busy_gauges.is_empty();
9536    let mut next_probe_at = Instant::now();
9537    let mut omission_counted = false;
9538
9539    loop {
9540        let now = Instant::now();
9541        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9542            let report = match scope {
9543                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9544                DrainScope::Endpoint(endpoint) => {
9545                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9546                }
9547            };
9548            gauges_quiescent = match report {
9549                Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9550                    BusyGaugeObservation::Quiescent => true,
9551                    BusyGaugeObservation::Busy => false,
9552                    BusyGaugeObservation::Omitted => {
9553                        if !omission_counted {
9554                            forwarding
9555                                .counters()
9556                                .increment_drains_with_undeclared_gauge();
9557                            omission_counted = true;
9558                        }
9559                        false
9560                    }
9561                },
9562                Err(err) => {
9563                    warn!(
9564                        module_id,
9565                        error = %err,
9566                        "drain health.check did not produce declared busy gauges; treating module as busy"
9567                    );
9568                    false
9569                }
9570            };
9571            // Counted from the probe's completion: the probe above was awaited,
9572            // so no second probe can start while one is still outstanding.
9573            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9574        }
9575
9576        let in_flight = forwarding
9577            .endpoint_in_flight_count(endpoint)
9578            .map_err(SuperviseError::Forwarding)?;
9579        if in_flight == 0 && gauges_quiescent {
9580            return Ok(true);
9581        }
9582
9583        let now = Instant::now();
9584        if now >= deadline {
9585            return Ok(false);
9586        }
9587        let mut wait = deadline
9588            .saturating_duration_since(now)
9589            .min(REGISTRY_RELEASE_POLL);
9590        if !busy_gauges.is_empty() {
9591            wait = wait.min(next_probe_at.saturating_duration_since(now));
9592        }
9593        sleep(wait).await;
9594    }
9595}
9596
9597/// The wait between drain busy-gauge probes: the larger of
9598/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
9599/// drain's own in-flight poll interval.
9600fn drain_busy_reprobe_interval() -> Duration {
9601    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9602}
9603
9604/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
9605///
9606/// `Ok` is always honest and passed straight through -- the wait actually measured
9607/// in-flight state. `Err` means the wait produced no measurement at all (the
9608/// forwarding table's lock was poisoned), so `false` is reported as the one honest
9609/// constant: the drain did not complete. Never recomputed from route state, never a
9610/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
9611fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9612    match wait_result {
9613        Ok(drained) => *drained,
9614        Err(_) => false,
9615    }
9616}
9617
9618fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9619    for released in released_routes {
9620        let frame = match Frame::build_with_version(
9621            released.negotiated_ver,
9622            FrameType::Goodbye,
9623            control_flags(),
9624            released.channel,
9625            released.epoch,
9626            0,
9627            Vec::new(),
9628        ) {
9629            Ok(frame) => frame,
9630            Err(err) => {
9631                warn!(
9632                    route_channel = released.channel,
9633                    error = %err,
9634                    "failed to build supervisor drain route GOODBYE frame"
9635                );
9636                continue;
9637            }
9638        };
9639        if !released.close_on_delivery_failure() {
9640            crate::forwarding::send_module_route_goodbye(
9641                &forwarding.counters(),
9642                &released.sink,
9643                frame,
9644                released.module_id.as_deref(),
9645                "supervisor drain",
9646            );
9647            continue;
9648        }
9649        if let Err(err) = released.sink.try_send(frame) {
9650            warn!(
9651                target_connection_id = released.connection_id.get(),
9652                route_channel = released.channel,
9653                error = %err,
9654                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9655            );
9656            let _ = forwarding.escalate_client_delivery_failure(
9657                released.connection_id,
9658                released.channel,
9659                released.epoch,
9660                CloseReason::new(
9661                    "route_goodbye_delivery_failed",
9662                    format!(
9663                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9664                        released.channel
9665                    ),
9666                ),
9667                crate::forwarding::UndeliveredFrame {
9668                    module_id: released.module_id.as_deref(),
9669                    sink: &released.sink,
9670                },
9671            );
9672        }
9673    }
9674}
9675
9676fn send_module_draining(
9677    module_id: &str,
9678    reason: RouteCloseReason,
9679    deadline_ms: u64,
9680    target: &ModuleDrainTarget,
9681) {
9682    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9683        reason,
9684        deadline_ms,
9685    }) {
9686        Ok(body) => body,
9687        Err(err) => {
9688            warn!(
9689                module_id,
9690                error = %err,
9691                "failed to encode module draining command"
9692            );
9693            return;
9694        }
9695    };
9696    let frame = match Frame::build_with_version(
9697        target.negotiated_ver,
9698        FrameType::Push,
9699        control_flags(),
9700        0,
9701        0,
9702        0,
9703        body,
9704    ) {
9705        Ok(frame) => frame,
9706        Err(err) => {
9707            warn!(
9708                module_id,
9709                error = %err,
9710                "failed to build module draining command frame"
9711            );
9712            return;
9713        }
9714    };
9715    if let Err(err) = target.sink.try_send(frame) {
9716        warn!(
9717            module_id,
9718            target_connection_id = target.endpoint.connection_id.get(),
9719            error = %err,
9720            "module draining command was not delivered to peer"
9721        );
9722    }
9723}
9724
9725/// The channel-0 GOODBYE that tells a module its stop is planned.
9726fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9727    match Frame::build_with_version(
9728        negotiated_ver,
9729        FrameType::Goodbye,
9730        control_flags(),
9731        0,
9732        0,
9733        0,
9734        Vec::new(),
9735    ) {
9736        Ok(frame) => Some(frame),
9737        Err(err) => {
9738            warn!(
9739                module_id,
9740                error = %err,
9741                "failed to build module GOODBYE frame"
9742            );
9743            None
9744        }
9745    }
9746}
9747
9748/// Send every registered module connection its module GOODBYE at daemon
9749/// shutdown, then request that connection's close.
9750///
9751/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
9752/// before EOF, so the GOODBYE must reach the socket before the close. A close
9753/// request does not wait for the connection's queued frames: its writer gets a
9754/// bounded grace after the close, is aborted if it overruns it, and the daemon
9755/// process may exit before that grace ends. So with `wait_for_flush`, each
9756/// connection is closed only after its writer has acknowledged writing the
9757/// GOODBYE, or once a short shared budget runs out, so one module that is not
9758/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
9759/// are only queued, for a shutdown the operator has told to stop waiting.
9760/// A connection that is already gone is skipped.
9761#[cfg(unix)]
9762async fn send_module_goodbyes_for_daemon_shutdown(
9763    forwarding: &Arc<ForwardingTable>,
9764    reason: &CloseReason,
9765    wait_for_flush: bool,
9766) {
9767    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9768    let targets = match forwarding.module_connections() {
9769        Ok(targets) => targets,
9770        Err(err) => {
9771            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9772            return;
9773        }
9774    };
9775    let deadline = Instant::now() + GOODBYE_BUDGET;
9776    let mut sends = tokio::task::JoinSet::new();
9777    for target in targets {
9778        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9779            continue;
9780        };
9781        if !wait_for_flush {
9782            if let Err(err) = target.sink.try_send(frame) {
9783                debug!(
9784                    module_id = %target.module_id,
9785                    error = %err,
9786                    "shutdown module GOODBYE was not queued"
9787                );
9788            }
9789            continue;
9790        }
9791        let forwarding = Arc::clone(forwarding);
9792        let reason = reason.clone();
9793        sends.spawn(async move {
9794            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9795                Ok(Ok(())) => {}
9796                Ok(Err(err)) => debug!(
9797                    module_id = %target.module_id,
9798                    error = %err,
9799                    "module connection closed before its shutdown GOODBYE was written"
9800                ),
9801                Err(_) => warn!(
9802                    module_id = %target.module_id,
9803                    budget = ?GOODBYE_BUDGET,
9804                    "shutdown module GOODBYE was not written within its budget; closing anyway"
9805                ),
9806            }
9807            forwarding.request_connection_close(target.endpoint.connection_id, reason);
9808        });
9809    }
9810    // Every task ends by the shared deadline, so this wait is bounded too.
9811    while sends.join_next().await.is_some() {}
9812}
9813
9814fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9815    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9816        return;
9817    };
9818    if let Err(err) = target.sink.try_send(frame) {
9819        warn!(
9820            module_id,
9821            target_connection_id = target.endpoint.connection_id.get(),
9822            error = %err,
9823            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9824        );
9825        forwarding.request_connection_close(
9826            target.endpoint.connection_id,
9827            CloseReason::new(
9828                "module_goodbye_delivery_failed",
9829                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9830            ),
9831        );
9832    }
9833}
9834
9835#[derive(Clone, Copy)]
9836struct ForwardingDrainContext<'a> {
9837    spec: &'a ModuleSpec,
9838    runtime: &'a SupervisorRuntimeConfig,
9839    registry: &'a Registry,
9840    scope: DrainScope,
9841}
9842
9843/// Which process a forwarding drain addresses.
9844#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9845enum DrainScope {
9846    /// Whatever endpoint is active for the module id: every plain stop,
9847    /// restart and reload. Also moves the module's state to `Draining`.
9848    Active,
9849    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
9850    /// module id would resolve to the promoted candidate and leave neither
9851    /// process routable. The module's state is left alone, since the promoted
9852    /// candidate is what it describes and that process is running.
9853    Endpoint(crate::ModuleEndpointId),
9854}
9855
9856/// Whether a child being drained has already been asked to stop by the time
9857/// its drain wait starts.
9858///
9859/// The drain wait is the same budget whatever this says. What it decides is
9860/// whether the supervisor must ask by signal before that wait begins: a child
9861/// that nobody asked will sit out the whole budget and then be SIGKILLed,
9862/// healthy or not.
9863#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9864enum StopNotice {
9865    /// The module was sent `module.draining` and a module GOODBYE over its own
9866    /// registered connection, and stops itself.
9867    SentOverConnection,
9868    /// The forwarding drain found no registered connection for the module: a
9869    /// subc child spawned moments ago that has not sent HELLO yet, or a
9870    /// `protocol: "none"` child, which never registers.
9871    NoConnection,
9872    /// This path sends nothing over the module's connection: the supervisor has
9873    /// no forwarding table, or the caller stops the child without a forwarding
9874    /// drain.
9875    NotSent,
9876}
9877
9878async fn begin_forwarding_drain(
9879    spec: &ModuleSpec,
9880    runtime: &SupervisorRuntimeConfig,
9881    registry: &Registry,
9882    snapshot: &SharedSnapshot,
9883    enabled: Option<bool>,
9884    reason: RouteCloseReason,
9885) -> Result<StopNotice, SuperviseError> {
9886    let Some(forwarding) = runtime.forwarding.as_ref() else {
9887        return Err(SuperviseError::ReloadUnavailable {
9888            module_id: spec.module_id.clone(),
9889            reason: "supervisor was not configured with a forwarding table".to_string(),
9890        });
9891    };
9892
9893    begin_forwarding_drain_with(
9894        forwarding,
9895        ForwardingDrainContext {
9896            spec,
9897            runtime,
9898            registry,
9899            scope: DrainScope::Active,
9900        },
9901        snapshot,
9902        enabled,
9903        reason,
9904        runtime.drain_timeout,
9905    )
9906    .await
9907}
9908
9909async fn begin_forwarding_drain_if_configured(
9910    spec: &ModuleSpec,
9911    runtime: &SupervisorRuntimeConfig,
9912    registry: &Registry,
9913    snapshot: &SharedSnapshot,
9914    enabled: Option<bool>,
9915    reason: RouteCloseReason,
9916) -> Result<StopNotice, SuperviseError> {
9917    begin_forwarding_drain_with_timeout(
9918        spec,
9919        runtime,
9920        registry,
9921        snapshot,
9922        enabled,
9923        reason,
9924        runtime.drain_timeout,
9925    )
9926    .await
9927}
9928
9929/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
9930/// budget, for paths where the operator overrides the module's configured one
9931/// (`supervisor.restart{drain_timeout_ms}`).
9932async fn begin_forwarding_drain_with_timeout(
9933    spec: &ModuleSpec,
9934    runtime: &SupervisorRuntimeConfig,
9935    registry: &Registry,
9936    snapshot: &SharedSnapshot,
9937    enabled: Option<bool>,
9938    reason: RouteCloseReason,
9939    drain_timeout: Duration,
9940) -> Result<StopNotice, SuperviseError> {
9941    let Some(forwarding) = runtime.forwarding.as_ref() else {
9942        return Ok(StopNotice::NotSent);
9943    };
9944
9945    begin_forwarding_drain_with(
9946        forwarding,
9947        ForwardingDrainContext {
9948            spec,
9949            runtime,
9950            registry,
9951            scope: DrainScope::Active,
9952        },
9953        snapshot,
9954        enabled,
9955        reason,
9956        drain_timeout,
9957    )
9958    .await
9959}
9960
9961async fn begin_forwarding_drain_with(
9962    forwarding: &ForwardingTable,
9963    context: ForwardingDrainContext<'_>,
9964    snapshot: &SharedSnapshot,
9965    enabled: Option<bool>,
9966    reason: RouteCloseReason,
9967    drain_timeout: Duration,
9968) -> Result<StopNotice, SuperviseError> {
9969    let ForwardingDrainContext {
9970        spec,
9971        runtime,
9972        registry,
9973        scope,
9974    } = context;
9975    debug_assert_ne!(reason, RouteCloseReason::Crash);
9976    let terminal = matches!(reason, RouteCloseReason::Disable);
9977    let drain_started_at = Instant::now();
9978    let drain_deadline = drain_started_at + drain_timeout;
9979    let deadline_ms =
9980        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9981    let busy_gauges = match scope {
9982        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9983        DrainScope::Endpoint(endpoint) => {
9984            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9985        }
9986    };
9987
9988    // Admission gate first: route.open/commit and route REQUEST admission are closed
9989    // before the first quiescence check, so the outstanding count can only fall.
9990    let gate_started = Instant::now();
9991    let drain_target = match scope {
9992        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9993        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9994    }
9995    .map_err(SuperviseError::Forwarding)?;
9996    // The instant admission closed, and how long taking the forwarding write
9997    // lock to close it took. The timeout line reports only the quiescence
9998    // wait, so without this a drain that started late looked like one that
9999    // started on time.
10000    info!(
10001        module_id = %spec.module_id,
10002        ?reason,
10003        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10004        connected = drain_target.is_some(),
10005        "module drain began; route admission closed"
10006    );
10007    if scope == DrainScope::Active {
10008        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10009            state.state = ModuleState::Draining;
10010            state.draining_to_replace =
10011                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10012            if let Some(enabled) = enabled {
10013                state.enabled = enabled;
10014            }
10015        })?;
10016    }
10017
10018    let Some(target) = drain_target.as_ref() else {
10019        // Nothing was sent: the module has no registered connection to carry
10020        // `module.draining` or a GOODBYE. The caller must not assume the child
10021        // was asked to stop.
10022        return Ok(StopNotice::NoConnection);
10023    };
10024    {
10025        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10026        #[cfg(feature = "test-support")]
10027        forwarding.wait_before_route_closing_for_test().await;
10028        let routes = forwarding
10029            .endpoint_routes(target.endpoint)
10030            .map_err(SuperviseError::Forwarding)?;
10031        let routes_notified = routes.len();
10032        crate::control::send_route_control_pushes(
10033            forwarding,
10034            routes.clone(),
10035            ClientControlPush::RouteClosing {
10036                module_id: spec.module_id.clone(),
10037                channels: Vec::new(),
10038                reason,
10039            },
10040        );
10041        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10042
10043        // `route.closing` was just sent above: from here on every return path,
10044        // including an early one, MUST send `route.closed` before propagating
10045        // anything else. A client holds `closing` as a promise that a verdict is
10046        // coming; leaving early without `closed` strands it waiting forever, since
10047        // `closing` carries no timeout of its own.
10048        let wait_result = wait_for_forwarding_quiescence(
10049            forwarding,
10050            &spec.module_id,
10051            runtime,
10052            target.endpoint,
10053            drain_deadline,
10054            &busy_gauges,
10055            scope,
10056        )
10057        .await;
10058        let drained = drained_after_quiescence_wait(&wait_result);
10059        if let Err(err) = &wait_result {
10060            error!(
10061                module_id = %spec.module_id,
10062                ?reason,
10063                error = %err,
10064                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10065            );
10066        } else if !drained {
10067            // Name what the drain waited on. Without it the line says only that
10068            // something did not settle, and "one wedged call" and "every
10069            // session's held stream" read the same; the first is a module bug,
10070            // the second is a module that should end its streams on
10071            // module.draining. Read before teardown releases the routes.
10072            let holdouts = forwarding
10073                .endpoint_drain_holdouts(target.endpoint)
10074                .unwrap_or_default();
10075            warn!(
10076                module_id = %spec.module_id,
10077                waited = ?drain_timeout,
10078                ?reason,
10079                held_requests = holdouts.requests,
10080                held_routes = holdouts.routes,
10081                total_routes = holdouts.total_routes,
10082                top_connections = ?holdouts.top_connections,
10083                // `module_channel:corr`, so the module can find each held request
10084                // in its own log; capped, so `held_requests` is the full count.
10085                held = %holdouts
10086                    .held
10087                    .iter()
10088                    .map(|(channel, corr)| format!("{channel}:{corr}"))
10089                    .collect::<Vec<_>>()
10090                    .join(","),
10091                "route drain timed out before request quiescence; forcing teardown"
10092            );
10093        }
10094        crate::control::send_route_control_pushes(
10095            forwarding,
10096            routes,
10097            ClientControlPush::RouteClosed {
10098                module_id: spec.module_id.clone(),
10099                channels: Vec::new(),
10100                reason,
10101                drained,
10102                abandoned: target.abandoned_bindings.len() as u32,
10103                excluded_subscriptions: target.excluded_subscriptions,
10104                terminal: Some(terminal),
10105            },
10106        );
10107        wait_result?;
10108
10109        // `route.closed` has now been sent unconditionally above. From here the
10110        // remaining steps are cleanup (route + module GOODBYE) rather than a
10111        // promise the client is waiting on, but a lock-poisoned
10112        // `release_module_endpoint_routes` would otherwise skip the module
10113        // GOODBYE silently too -- send it before propagating the error.
10114        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10115            Ok(routes) => routes,
10116            Err(err) => {
10117                warn!(
10118                    module_id = %spec.module_id,
10119                    ?reason,
10120                    error = %err,
10121                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10122                );
10123                send_module_goodbye(&spec.module_id, forwarding, target);
10124                return Err(SuperviseError::Forwarding(err));
10125            }
10126        };
10127        let route_goodbye_count = released_routes.len();
10128        send_route_goodbyes(forwarding, released_routes);
10129        send_module_goodbye(&spec.module_id, forwarding, target);
10130
10131        // The drain's happy path was previously silent: every emission above is
10132        // best-effort with only its failure arm logged, so "were consumers told"
10133        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10134        // hang where the open question was exactly whether teardown notice went
10135        // out). One summary line makes that class decidable in one grep.
10136        info!(
10137            module_id = %spec.module_id,
10138            ?reason,
10139            routes_notified,
10140            route_goodbyes = route_goodbye_count,
10141            abandoned_reservations = target.abandoned_bindings.len(),
10142            excluded_subscriptions = target.excluded_subscriptions,
10143            drained,
10144            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10145        );
10146    }
10147
10148    Ok(StopNotice::SentOverConnection)
10149}
10150
10151/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10152/// the only slot a plain (non-swap) spawn can register into.
10153async fn wait_for_registration_after_reload(
10154    registry: &Registry,
10155    module_id: &str,
10156    snapshot: &SharedSnapshot,
10157    child: &mut SupervisedChild,
10158    wait: Duration,
10159) -> Result<RegistrationWaitOutcome, SuperviseError> {
10160    wait_for_slot_registration(
10161        registry,
10162        crate::registry::RegistrationSlot::Active(module_id),
10163        module_id,
10164        snapshot,
10165        child,
10166        wait,
10167    )
10168    .await
10169}
10170
10171/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10172///
10173/// Keyed on the slot rather than the bare module id because during a swap the
10174/// id's active slot is already held by the incumbent: an id-keyed wait would
10175/// report the incumbent's registration as the candidate's and a candidate that
10176/// never registers would look registered. A swap candidate waits on
10177/// `crate::registry::RegistrationSlot::Candidate`.
10178async fn wait_for_slot_registration(
10179    registry: &Registry,
10180    slot: crate::registry::RegistrationSlot<'_>,
10181    module_id: &str,
10182    snapshot: &SharedSnapshot,
10183    child: &mut SupervisedChild,
10184    wait: Duration,
10185) -> Result<RegistrationWaitOutcome, SuperviseError> {
10186    let deadline = Instant::now() + wait;
10187    loop {
10188        if registry
10189            .registration(slot)
10190            .map_err(SuperviseError::Registry)?
10191            .is_some()
10192        {
10193            return Ok(RegistrationWaitOutcome::Registered);
10194        }
10195
10196        let now = Instant::now();
10197        if now >= deadline {
10198            return Ok(RegistrationWaitOutcome::TimedOut);
10199        }
10200        let remaining = deadline.saturating_duration_since(now);
10201        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10202
10203        tokio::select! {
10204            wait_result = child.wait() => {
10205                let status = wait_result.map_err(|source| SuperviseError::Wait {
10206                    module_id: module_id.to_string(),
10207                    source,
10208                })?;
10209                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10210                    snapshot,
10211                    child,
10212                    &status,
10213                )));
10214            }
10215            _ = sleep(poll) => {}
10216        }
10217    }
10218}
10219
10220fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10221    // A replacement process that exits before HELLO did not provide service, even
10222    // if it used status 0. Count it against the restart cap as a new-binary failure.
10223    if exit_report.kind != ExitKind::DeliberateSeverance {
10224        exit_report.kind = ExitKind::Crash;
10225    }
10226    exit_report
10227}
10228
10229async fn handle_reload_child_registration_failure(
10230    spec: &ModuleSpec,
10231    runtime: &SupervisorRuntimeConfig,
10232    registry: &Registry,
10233    process_liveness: &SupervisorProcessLiveness,
10234    snapshot: &SharedSnapshot,
10235    _child: &mut Option<SupervisedChild>,
10236    failure: ReloadRegistrationFailure,
10237) -> Result<(), SuperviseError> {
10238    let ReloadRegistrationFailure {
10239        exit_report,
10240        reason,
10241    } = failure;
10242    match on_child_exit(
10243        spec,
10244        runtime.restart_policy,
10245        registry,
10246        snapshot,
10247        &runtime.terminal_ring,
10248        &runtime.spawn_events,
10249        &runtime.child_roster,
10250        exit_report,
10251    )
10252    .await
10253    {
10254        NextAction::Stop {
10255            registration_released,
10256        } => {
10257            if registration_released {
10258                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10259            }
10260        }
10261        NextAction::Restart { schedule } => {
10262            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10263                schedule.delay
10264            });
10265            if let Some(schedule) = schedule {
10266                log_crash_respawn(&spec.module_id, schedule);
10267            }
10268            schedule_respawn(
10269                runtime,
10270                snapshot,
10271                &spec.module_id,
10272                delay,
10273                RespawnKind::Spawn,
10274            )?;
10275        }
10276    }
10277    Err(SuperviseError::ReloadFailed {
10278        module_id: spec.module_id.clone(),
10279        reason,
10280    })
10281}
10282
10283async fn handle_reload_spawn_failure(
10284    spec: &ModuleSpec,
10285    runtime: &SupervisorRuntimeConfig,
10286    process_liveness: &SupervisorProcessLiveness,
10287    snapshot: &SharedSnapshot,
10288    _child: &mut Option<SupervisedChild>,
10289    reason: String,
10290) -> Result<(), SuperviseError> {
10291    let now = Instant::now();
10292    let mut schedule = None;
10293    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10294        clear_current_process_facts(state);
10295        if state.enabled {
10296            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10297            state.state = if schedule.is_some() {
10298                ModuleState::Restarting
10299            } else {
10300                ModuleState::Failed
10301            };
10302        } else {
10303            state.state = ModuleState::Disabled;
10304        }
10305    })?;
10306    if let Some(schedule) = schedule {
10307        schedule_respawn(
10308            runtime,
10309            snapshot,
10310            &spec.module_id,
10311            schedule.delay,
10312            RespawnKind::Spawn,
10313        )?;
10314    } else {
10315        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10316    }
10317    Err(SuperviseError::ReloadFailed {
10318        module_id: spec.module_id.clone(),
10319        reason,
10320    })
10321}
10322
10323fn control_flags() -> Flags {
10324    Flags::new(false, Priority::Passive, false)
10325}
10326
10327#[allow(clippy::too_many_arguments)]
10328async fn drain_optional_child(
10329    module_id: &str,
10330    protocol: ModuleProtocol,
10331    stop_notice: StopNotice,
10332    registry: &Registry,
10333    forwarding: Option<&ForwardingTable>,
10334    snapshot: &SharedSnapshot,
10335    terminal_ring: &Arc<Mutex<TerminalRing>>,
10336    spawn_events: &SpawnEventFeed,
10337    child: &mut Option<SupervisedChild>,
10338    drain_timeout: Duration,
10339    final_state: ModuleState,
10340    enabled: Option<bool>,
10341) -> Result<(), SuperviseError> {
10342    if let Some(child) = child.take() {
10343        drain_child_to_state(
10344            module_id,
10345            protocol,
10346            stop_notice,
10347            registry,
10348            forwarding,
10349            snapshot,
10350            terminal_ring,
10351            spawn_events,
10352            child,
10353            drain_timeout,
10354            final_state,
10355            enabled,
10356        )
10357        .await
10358    } else {
10359        update_snapshot(snapshot, Some(module_id), |state| {
10360            state.state = final_state;
10361            if let Some(enabled) = enabled {
10362                state.enabled = enabled;
10363            }
10364            clear_current_process_facts(state);
10365        })?;
10366        release_dead_registration(registry, forwarding, snapshot, module_id).await
10367    }
10368}
10369
10370#[allow(clippy::too_many_arguments)]
10371async fn drain_child_to_state(
10372    module_id: &str,
10373    _protocol: ModuleProtocol,
10374    stop_notice: StopNotice,
10375    registry: &Registry,
10376    forwarding: Option<&ForwardingTable>,
10377    snapshot: &SharedSnapshot,
10378    terminal_ring: &Arc<Mutex<TerminalRing>>,
10379    spawn_events: &SpawnEventFeed,
10380    mut child: SupervisedChild,
10381    drain_timeout: Duration,
10382    final_state: ModuleState,
10383    enabled: Option<bool>,
10384) -> Result<(), SuperviseError> {
10385    let protocol = child.protocol;
10386    update_snapshot(snapshot, Some(module_id), |state| {
10387        state.state = ModuleState::Draining;
10388        state.draining_to_replace = final_state == ModuleState::Restarting;
10389        if let Some(enabled) = enabled {
10390            state.enabled = enabled;
10391        }
10392    })?;
10393
10394    // The wait below is the same budget in every case; what differs is
10395    // whether anything has ASKED the child to stop before it starts. Only a
10396    // forwarding drain that reached the module's registered connection has
10397    // (`module.draining`, then a module GOODBYE). Every other child was told
10398    // nothing: a `protocol: "none"` module, which never registers; a subc
10399    // module spawned moments ago that has not sent HELLO yet; or a stop that
10400    // runs no forwarding drain. Without a signal the budget is only a delay
10401    // in front of SIGKILL -- and the not-yet-registered child is the worst
10402    // case, because it registers into a module that is already draining,
10403    // is never told, and is killed while healthy.
10404    if stop_notice != StopNotice::SentOverConnection {
10405        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10406            info!(
10407                module_id,
10408                pid = child.pid,
10409                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10410                "module has no connection yet; requesting stop by signal"
10411            );
10412        }
10413        request_graceful_stop(module_id, &child);
10414    }
10415
10416    let exit_report = match timeout(drain_timeout, child.wait()).await {
10417        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10418        Ok(Err(source)) => {
10419            fail_after_wait_error(
10420                module_id,
10421                snapshot,
10422                terminal_ring,
10423                spawn_events,
10424                format!("failed to wait during drain: {source}"),
10425            );
10426            return Err(SuperviseError::Wait {
10427                module_id: module_id.to_string(),
10428                source,
10429            });
10430        }
10431        Err(_) => {
10432            // Mirror the sibling arm above: state is already `Draining`, and an
10433            // error propagated from here would strand it there -- a state
10434            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10435            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10436            // `Failed` before `?` keeps the module operator-visible and
10437            // revivable. Trigger is an ESRCH race (process exits between the
10438            // drain timeout firing and the kill) or a post-kill wait failure
10439            // (issue #34).
10440            //
10441            // Logged because the kill is otherwise visible only as signal 9 in
10442            // the terminal ring, and the budget it follows can be long enough
10443            // that consumers see a stretch of refusals with no stated cause.
10444            warn!(
10445                module_id,
10446                pid = child.pid,
10447                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10448                reason = ?final_state,
10449                ?stop_notice,
10450                "drain budget expired before the module exited; killing it"
10451            );
10452            child.start_kill().map_err(|source| {
10453                fail_after_wait_error(
10454                    module_id,
10455                    snapshot,
10456                    terminal_ring,
10457                    spawn_events,
10458                    format!("failed to kill during drain: {source}"),
10459                );
10460                SuperviseError::Kill {
10461                    module_id: module_id.to_string(),
10462                    source,
10463                }
10464            })?;
10465            let status = child.wait().await.map_err(|source| {
10466                fail_after_wait_error(
10467                    module_id,
10468                    snapshot,
10469                    terminal_ring,
10470                    spawn_events,
10471                    format!("failed to wait after drain kill: {source}"),
10472                );
10473                SuperviseError::Wait {
10474                    module_id: module_id.to_string(),
10475                    source,
10476                }
10477            })?;
10478            classify_reaped_child_exit(snapshot, &child, &status)
10479        }
10480    };
10481
10482    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10483    publish_after_terminal_record(
10484        module_id,
10485        || {
10486            record_terminal_with_detail(
10487                module_id,
10488                terminal_ring,
10489                spawn_events,
10490                &exit_report,
10491                terminal_disposition(final_state),
10492                detail,
10493            )
10494        },
10495        || {
10496            update_snapshot(snapshot, Some(module_id), |state| {
10497                state.state = final_state;
10498                if let Some(enabled) = enabled {
10499                    state.enabled = enabled;
10500                }
10501                clear_current_process_facts(state);
10502                state.last_exit = Some(exit_report.clone());
10503                if exit_report.kind == ExitKind::DeliberateSeverance {
10504                    state.lifetime_restarts += 1;
10505                }
10506            })
10507        },
10508    )?;
10509    child.drain_stderr(module_id).await;
10510
10511    release_dead_registration(registry, forwarding, snapshot, module_id).await
10512}
10513
10514/// Ask a child that nothing else has asked to stop, by signal.
10515///
10516/// A registered subc module is asked over its own connection: the drain sends
10517/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10518/// module GOODBYE, and the module stops itself. A module that speaks no subc
10519/// wire receives none of that, and neither does a subc module that has not
10520/// registered yet, so for them the drain budget would be pure delay in front of
10521/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10522/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10523/// into a recovery on the next start.
10524///
10525/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10526/// rule rather than an optimisation: that module's graceful stop is already
10527/// running by the time its child is drained, and a signal would race it.
10528///
10529/// Best-effort by construction. A child that has already exited is the ordinary
10530/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10531/// failure is logged at debug and the wait-then-kill below still decides the
10532/// outcome.
10533#[cfg(unix)]
10534fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10535    let Some(pid) = child
10536        .id()
10537        .and_then(|pid| i32::try_from(pid).ok())
10538        .and_then(rustix::process::Pid::from_raw)
10539    else {
10540        debug!(
10541            module_id,
10542            "no pid to signal for teardown; falling through to the drain wait"
10543        );
10544        return;
10545    };
10546    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10547        Ok(()) => debug!(
10548            module_id,
10549            "sent SIGTERM to a module nothing else asked to stop"
10550        ),
10551        Err(err) => debug!(
10552            module_id,
10553            error = %err,
10554            "SIGTERM to module failed; the drain wait and kill still apply"
10555        ),
10556    }
10557}
10558
10559/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
10560/// Windows does offer need cooperation this supervisor cannot assume: a console
10561/// control event requires sharing a console with the child, and `WM_CLOSE`
10562/// requires the child to pump a message loop. A supervised server process does
10563/// neither, so there is nothing to send and teardown is the wait followed by the
10564/// kill. Emulating a signal here would mean inventing a stop protocol, which is
10565/// the thing `protocol: "none"` exists to avoid.
10566#[cfg(not(unix))]
10567fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10568    debug!(
10569        module_id,
10570        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10571    );
10572}
10573
10574fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10575    match final_state {
10576        ModuleState::Stopped => TerminalDisposition::Stopped,
10577        ModuleState::Disabled => TerminalDisposition::Disabled,
10578        ModuleState::Restarting => TerminalDisposition::Restarting,
10579        ModuleState::Failed => TerminalDisposition::Failed,
10580        ModuleState::Starting
10581        | ModuleState::Running
10582        | ModuleState::Unresponsive
10583        | ModuleState::Draining => {
10584            unreachable!("terminal exits only finish in terminal or restarting states")
10585        }
10586    }
10587}
10588
10589/// Release a reaped child's registration before allowing another spawn.
10590///
10591/// EOF is not a process-lifetime signal: an inherited socket can stay open
10592/// indefinitely, and serial frame dispatch can be waiting on egress instead of
10593/// reading EOF. After the normal release grace, request connection close (which
10594/// cancels both reads and dispatch), then allow one more release grace for the
10595/// connection guard's forwarding cleanup. Never evict a different connection.
10596async fn release_dead_registration(
10597    registry: &Registry,
10598    forwarding: Option<&ForwardingTable>,
10599    snapshot: &SharedSnapshot,
10600    module_id: &str,
10601) -> Result<(), SuperviseError> {
10602    let result = async {
10603        let registration = registry
10604            .get_module(module_id)
10605            .map_err(SuperviseError::Registry)?;
10606        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10607            Ok(()) => return Ok(()),
10608            Err(SuperviseError::RegistrationStillActive { .. }) => {}
10609            Err(err) => return Err(err),
10610        }
10611        let pid = lock_snapshot(snapshot)?.reaped_pid;
10612        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10613            warn!(
10614                module_id,
10615                pid,
10616                connection_id = registration.connection_id.get(),
10617                "reaped module registration outlived release grace; closing dead connection"
10618            );
10619            forwarding.request_connection_close(
10620                registration.connection_id,
10621                CloseReason::new(
10622                    "supervised_process_reaped",
10623                    format!("module '{module_id}' pid {pid} exited"),
10624                ),
10625            );
10626            wait_for_slot_registration_release(
10627                registry,
10628                crate::registry::RegistrationSlot::Connection(registration.connection_id),
10629                REGISTRY_RELEASE_TIMEOUT,
10630            )
10631            .await?;
10632        }
10633        wait_for_registration_release(registry, module_id, Duration::ZERO).await
10634    }
10635    .await;
10636    if let Err(err) = &result {
10637        fail_snapshot(snapshot, Some(module_id), None);
10638        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10639    }
10640    result
10641}
10642
10643/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
10644/// plain stop or restart waits for before it spawns a replacement.
10645async fn wait_for_registration_release(
10646    registry: &Registry,
10647    module_id: &str,
10648    wait: Duration,
10649) -> Result<(), SuperviseError> {
10650    wait_for_slot_registration_release(
10651        registry,
10652        crate::registry::RegistrationSlot::Active(module_id),
10653        wait,
10654    )
10655    .await
10656}
10657
10658/// Wait for the registration in `slot` to go away.
10659///
10660/// Keyed on the slot rather than the bare module id because a successful swap
10661/// never empties the id's active slot (the promoted candidate is in it), so an
10662/// id-keyed wait for the incumbent's release would always time out. Draining a
10663/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
10664/// incumbent's connection instead.
10665async fn wait_for_slot_registration_release(
10666    registry: &Registry,
10667    slot: crate::registry::RegistrationSlot<'_>,
10668    wait: Duration,
10669) -> Result<(), SuperviseError> {
10670    let deadline = Instant::now() + wait;
10671    let mut release_events = registration_release_events().subscribe();
10672    let still_active = |registration: &crate::registry::ModuleRegistration| {
10673        SuperviseError::RegistrationStillActive {
10674            module_id: registration.manifest.module_id.clone(),
10675            waited: wait,
10676        }
10677    };
10678    loop {
10679        let _observed_generation = *release_events.borrow_and_update();
10680        let Some(registration) = registry
10681            .registration(slot)
10682            .map_err(SuperviseError::Registry)?
10683        else {
10684            return Ok(());
10685        };
10686
10687        let now = Instant::now();
10688        if now >= deadline {
10689            return Err(still_active(&registration));
10690        }
10691
10692        let remaining = deadline.saturating_duration_since(now);
10693        match timeout(remaining, release_events.changed()).await {
10694            Ok(Ok(())) | Ok(Err(_)) => {}
10695            Err(_) => return Err(still_active(&registration)),
10696        }
10697    }
10698}
10699
10700#[cfg(test)]
10701mod slot_registration_wait_tests {
10702    use super::*;
10703    use crate::registry::{ConnectionId, RegistrationSlot};
10704    use subc_protocol::manifest::ModuleManifest;
10705
10706    #[tokio::test]
10707    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10708        let registry = Arc::new(Registry::default());
10709        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
10710        let runtime = supervisor.runtime_config();
10711        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10712        let spec = ModuleSpec {
10713            module_id: "enable-stale-registration".to_string(),
10714            program: PathBuf::from("/missing/enable-retry-test"),
10715            args: Vec::new(),
10716            env: Vec::new(),
10717            reserved: false,
10718            reserved_prefixes: Vec::new(),
10719            protocol: ModuleProtocol::Subc,
10720            overlap: Default::default(),
10721        };
10722        let connection = ConnectionId::new(90);
10723        registry
10724            .register_with_control_ops(
10725                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10726                1,
10727                connection,
10728                Vec::new(),
10729            )
10730            .unwrap();
10731        let mut child = None;
10732        let err = set_child_enabled(
10733            &spec,
10734            &runtime,
10735            &registry,
10736            &supervisor.process_liveness,
10737            &snapshot,
10738            &mut child,
10739            true,
10740        )
10741        .await
10742        .unwrap_err();
10743        assert!(matches!(
10744            err,
10745            SuperviseError::RegistrationStillActive { .. }
10746        ));
10747        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10748        assert!(child.is_none());
10749        registry.deregister_connection(connection).unwrap();
10750        let err = set_child_enabled(
10751            &spec,
10752            &runtime,
10753            &registry,
10754            &supervisor.process_liveness,
10755            &snapshot,
10756            &mut child,
10757            true,
10758        )
10759        .await
10760        .unwrap_err();
10761        assert!(
10762            matches!(err, SuperviseError::Spawn { .. }),
10763            "second enable must attempt a spawn: {err}"
10764        );
10765        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10766    }
10767
10768    const INCUMBENT: u64 = 1;
10769    const CANDIDATE: u64 = 2;
10770
10771    fn swapped_registry() -> Arc<Registry> {
10772        let registry = Arc::new(Registry::default());
10773        let manifest = ModuleManifest::builder("m", "0.1.0").build();
10774        registry
10775            .register_with_control_ops(
10776                manifest.clone(),
10777                1,
10778                ConnectionId::new(INCUMBENT),
10779                Vec::new(),
10780            )
10781            .unwrap();
10782        registry
10783            .register_candidate_with_control_ops(
10784                manifest,
10785                1,
10786                ConnectionId::new(CANDIDATE),
10787                Vec::new(),
10788            )
10789            .unwrap();
10790        registry
10791    }
10792
10793    /// After a promotion the id's active slot is held by the new process, so an
10794    /// id-keyed wait for the incumbent's release can never succeed; the
10795    /// connection-keyed wait completes as soon as the incumbent deregisters.
10796    #[tokio::test]
10797    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10798        let registry = swapped_registry();
10799        registry.promote_candidate("m").unwrap().unwrap();
10800
10801        assert!(matches!(
10802            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
10803            Err(SuperviseError::RegistrationStillActive { .. })
10804        ));
10805
10806        // Still held while the incumbent's connection has not deregistered.
10807        assert!(matches!(
10808            wait_for_slot_registration_release(
10809                &registry,
10810                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10811                Duration::from_millis(50),
10812            )
10813            .await,
10814            Err(SuperviseError::RegistrationStillActive { .. })
10815        ));
10816
10817        let releaser = Arc::clone(&registry);
10818        let release = tokio::spawn(async move {
10819            sleep(Duration::from_millis(20)).await;
10820            releaser
10821                .deregister_connection(ConnectionId::new(INCUMBENT))
10822                .unwrap();
10823            notify_registration_release();
10824        });
10825        wait_for_slot_registration_release(
10826            &registry,
10827            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10828            Duration::from_secs(5),
10829        )
10830        .await
10831        .expect("the incumbent's own registration is released");
10832        release.await.unwrap();
10833        assert!(registry.get_module("m").unwrap().is_some());
10834    }
10835
10836    /// The candidate slot is waited on separately from the active slot: the
10837    /// incumbent's registration neither holds up nor stands in for it.
10838    #[tokio::test]
10839    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10840        let registry = swapped_registry();
10841        assert!(matches!(
10842            wait_for_slot_registration_release(
10843                &registry,
10844                RegistrationSlot::Candidate("m"),
10845                Duration::from_millis(50),
10846            )
10847            .await,
10848            Err(SuperviseError::RegistrationStillActive { .. })
10849        ));
10850        registry
10851            .deregister_connection(ConnectionId::new(CANDIDATE))
10852            .unwrap();
10853        wait_for_slot_registration_release(
10854            &registry,
10855            RegistrationSlot::Candidate("m"),
10856            Duration::from_millis(50),
10857        )
10858        .await
10859        .expect("a candidate slot with no candidate is released");
10860        assert!(registry
10861            .registration(RegistrationSlot::Active("m"))
10862            .unwrap()
10863            .is_some());
10864    }
10865}
10866
10867fn classify_exit(status: &ExitStatus) -> ExitReport {
10868    ExitReport {
10869        kind: if status.success() {
10870            ExitKind::Clean
10871        } else {
10872            ExitKind::Crash
10873        },
10874        code: status.code(),
10875        signal: exit_signal(status),
10876        at_ms: unix_ms_now(),
10877    }
10878}
10879
10880/// The terminal record for a module whose `wait()` call itself errored (e.g. the
10881/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
10882/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
10883/// disposition still must be `Failed` so the terminal ring is not silently missing
10884/// an entry, matching what `fail_snapshot` records for this same arm.
10885fn wait_error_exit_report() -> ExitReport {
10886    ExitReport {
10887        kind: ExitKind::Crash,
10888        code: None,
10889        signal: None,
10890        at_ms: unix_ms_now(),
10891    }
10892}
10893
10894#[cfg(unix)]
10895fn exit_signal(status: &ExitStatus) -> Option<i32> {
10896    use std::os::unix::process::ExitStatusExt;
10897
10898    status.signal()
10899}
10900
10901#[cfg(not(unix))]
10902fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10903    None
10904}
10905
10906/// Give an operator-touched module its full crash budget back.
10907///
10908/// Named for the counter it used to zero; it now empties the in-window ring,
10909/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
10910/// ledger of what happened survives every operator action.
10911fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10912    update_snapshot(snapshot, Some(module_id), |state| {
10913        state.clear_crash_restarts();
10914    })
10915}
10916
10917fn set_running(
10918    snapshot: &SharedSnapshot,
10919    child: &SupervisedChild,
10920    module_id: &str,
10921    spawn_events: &SpawnEventFeed,
10922) -> Result<(), SuperviseError> {
10923    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10924        module_id: Some(module_id.to_string()),
10925    })?;
10926    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10927    if std::mem::take(&mut state.coalesced_restart_pending) {
10928        let generation = state.spawn_generation;
10929        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10930    }
10931    state.drain_disposition_detail = None;
10932    state.spawn_failure = None;
10933    state.waiting_for_previous_pid = None;
10934    state.waiting_for_previous_detail = None;
10935    // Every caller of this is a plain spawn, which always uses the primary key;
10936    // a promoted swap candidate sets the flag itself after this returns.
10937    state.in_alternate_slot = false;
10938    state.configuration_updated_since_spawn = false;
10939    state.spawned_protocol = Some(child.protocol);
10940    state.state = ModuleState::Running;
10941    state.enabled = true;
10942    state.process_alive = true;
10943    state.pid = child.id();
10944    #[cfg(windows)]
10945    {
10946        state.pipe_consumed = child
10947            .nonce_delivery
10948            .as_ref()
10949            .map(|delivery| delivery.consumed());
10950    }
10951    #[cfg(target_os = "macos")]
10952    {
10953        state.report_ready = Some(Arc::clone(&child.report_ready));
10954    }
10955    state.spawned_at_ms = Some(child.spawned_at_ms);
10956    state.exec_outcome = Some(Arc::clone(&child.exec.0));
10957    state.spawned_from = Some(child.spawned_from.clone());
10958    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
10959    state.spawned_file_identity = child.spawned_file_identity.clone();
10960    state.process_start_time = child.process_start_time;
10961    Ok(())
10962}
10963
10964fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10965    state.waiting_for_previous_pid = None;
10966    state.waiting_for_previous_detail = None;
10967    state.process_alive = false;
10968    state.spawned_protocol = None;
10969    state.pid = None;
10970    #[cfg(windows)]
10971    {
10972        state.pipe_consumed = None;
10973    }
10974    #[cfg(target_os = "macos")]
10975    {
10976        state.report_ready = None;
10977    }
10978    state.spawned_at_ms = None;
10979    state.exec_outcome = None;
10980    state.spawned_from = None;
10981    state.launch_spec_digest = None;
10982    state.spawned_file_identity = None;
10983    state.process_start_time = None;
10984    state.deliberate_severance = None;
10985}
10986
10987#[cfg(test)]
10988fn record_deliberate_severance(
10989    snapshot: &SharedSnapshot,
10990    identity: ProcessIdentity,
10991) -> Result<(), SuperviseError> {
10992    update_snapshot(snapshot, None, |state| {
10993        state.deliberate_severance = Some(identity);
10994    })
10995}
10996
10997fn apply_deliberate_severance_marker(
10998    snapshot: &SharedSnapshot,
10999    exited_identity: Option<ProcessIdentity>,
11000    mut exit_report: ExitReport,
11001) -> ExitReport {
11002    let marker = lock_snapshot(snapshot)
11003        .ok()
11004        .and_then(|mut state| state.deliberate_severance.take());
11005    if marker.is_some() && marker == exited_identity {
11006        exit_report.kind = ExitKind::DeliberateSeverance;
11007    }
11008    exit_report
11009}
11010
11011fn classify_reaped_child_exit(
11012    snapshot: &SharedSnapshot,
11013    child: &SupervisedChild,
11014    status: &ExitStatus,
11015) -> ExitReport {
11016    let _ = update_snapshot(snapshot, None, |state| {
11017        state.reaped_pid = Some(child.pid);
11018        state.spawn_failure = child.spawn_failure.clone();
11019    });
11020    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11021}
11022
11023fn fail_snapshot(
11024    snapshot: &SharedSnapshot,
11025    module_id: Option<&str>,
11026    last_exit: Option<ExitReport>,
11027) {
11028    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11029        state.state = ModuleState::Failed;
11030        clear_current_process_facts(state);
11031        if let Some(last_exit) = last_exit {
11032            state.last_exit = Some(last_exit);
11033        }
11034    }) {
11035        error!(error = %err, "failed to mark supervisor state failed");
11036    }
11037}
11038
11039fn update_snapshot(
11040    snapshot: &SharedSnapshot,
11041    module_id: Option<&str>,
11042    update: impl FnOnce(&mut SupervisorSnapshot),
11043) -> Result<(), SuperviseError> {
11044    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11045        module_id: module_id.map(ToOwned::to_owned),
11046    })?;
11047    update(&mut state);
11048    Ok(())
11049}
11050
11051const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11052
11053fn lock_snapshot_for_control<'a>(
11054    snapshot: &'a SharedSnapshot,
11055    module_id: &str,
11056    caller: &'static str,
11057) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11058    let started_at = Instant::now();
11059    let guard = lock_snapshot(snapshot)?;
11060    let waited = started_at.elapsed();
11061    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11062        warn!(
11063            module_id = %module_id,
11064            waited_ms = waited.as_millis() as u64,
11065            caller = %caller,
11066            "slow snapshot lock"
11067        );
11068    }
11069    Ok(guard)
11070}
11071
11072fn lock_snapshot(
11073    snapshot: &SharedSnapshot,
11074) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11075    snapshot
11076        .lock()
11077        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11078}
11079
11080#[cfg(test)]
11081mod terminal_history_tests {
11082    use std::{
11083        path::PathBuf,
11084        sync::Arc,
11085        time::{Duration, Instant},
11086    };
11087
11088    use tokio::{sync::mpsc, time::sleep};
11089
11090    use super::{
11091        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11092        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11093        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11094        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11095        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11096        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11097        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11098        SupervisorHealthStatus, SupervisorSnapshot,
11099    };
11100    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11101    // use for their own wall-clock deadlines: crash-restart instants must be on
11102    // the same clock the production code stamps them with, which is tokio's (and
11103    // is what `start_paused` tests can move).
11104    use super::Instant as ClockInstant;
11105    use crate::{
11106        registry::Registry,
11107        terminal_ring::{TerminalRing, TerminalRingConfig},
11108    };
11109    use std::sync::Mutex;
11110    use subc_control::TerminalDisposition;
11111
11112    /// See the twin in `control.rs` for why this derives the path from
11113    /// `current_exe()` and why the existence check is here: `--lib` alone does
11114    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11115    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11116    pub(super) fn fake_aft_stub_path() -> PathBuf {
11117        let mut path = std::env::current_exe().expect("current_exe available in tests");
11118        path.pop();
11119        path.pop();
11120        path.push(if cfg!(windows) {
11121            "fake-aft-stub.exe"
11122        } else {
11123            "fake-aft-stub"
11124        });
11125        assert!(
11126            path.exists(),
11127            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11128             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11129            path.display()
11130        );
11131        path
11132    }
11133
11134    #[test]
11135    fn reserved_never_spawned_refuses_every_hello() {
11136        // The canary hole: a reserved id whose module has never spawned had NO
11137        // gate entry and admitted anyone -- the reservation protected the nonce
11138        // holder, not the NAME. Now the entry is present with no legitimate
11139        // holder and refuses all comers.
11140        let supervisor = SupervisorHandle::default();
11141        supervisor.apply_identity_configuration(&ModuleSpec {
11142            module_id: "never-spawned".to_string(),
11143            program: PathBuf::from("/usr/bin/false"),
11144            args: Vec::new(),
11145            env: Vec::new(),
11146            reserved: true,
11147            reserved_prefixes: Vec::new(),
11148            protocol: ModuleProtocol::Subc,
11149            overlap: Default::default(),
11150        });
11151        assert!(
11152            supervisor
11153                .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11154                .is_some(),
11155            "forged nonce must refuse on a reserved never-spawned id"
11156        );
11157        assert!(
11158            supervisor
11159                .reserved_hello_rejection("never-spawned", None)
11160                .is_some(),
11161            "absent nonce must refuse on a reserved never-spawned id"
11162        );
11163        // And a real spawn nonce minted later admits exactly that nonce.
11164        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11165        supervisor.apply_identity_configuration(&ModuleSpec {
11166            module_id: "never-spawned".to_string(),
11167            program: PathBuf::from("/usr/bin/false"),
11168            args: Vec::new(),
11169            env: Vec::new(),
11170            reserved: true,
11171            reserved_prefixes: Vec::new(),
11172            protocol: ModuleProtocol::Subc,
11173            overlap: Default::default(),
11174        });
11175        assert!(supervisor
11176            .reserved_hello_rejection("never-spawned", Some("minted"))
11177            .is_none());
11178        assert!(supervisor
11179            .reserved_hello_rejection("never-spawned", Some("forged"))
11180            .is_some());
11181    }
11182
11183    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11184    /// happened, which is what "spent budget" looks like to every reader.
11185    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11186        let now = ClockInstant::now();
11187        for _ in 0..count {
11188            state.crash_restarts.push_back(now);
11189        }
11190    }
11191
11192    /// Age the oldest recorded restart out of `window`, standing in for the hours
11193    /// that would otherwise have to pass. Injecting the instant is the point: a
11194    /// test that slept a real window would take ten minutes and still prove less.
11195    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11196        let aged = state
11197            .crash_restarts
11198            .front()
11199            .expect("a crash restart must be recorded before it can be aged")
11200            .checked_sub(window + Duration::from_secs(1))
11201            .expect("the test clock is far enough from its origin to age an instant");
11202        state.crash_restarts[0] = aged;
11203    }
11204
11205    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11206        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11207        seed_crash_restarts(&mut state, count);
11208        state
11209    }
11210
11211    #[test]
11212    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11213        let policy = RestartPolicy::new(3, Duration::ZERO);
11214        let now = ClockInstant::now();
11215        assert!(daemon_will_restart(
11216            &mut snapshot_with_restarts(true, 2),
11217            &policy,
11218            now
11219        ));
11220        assert!(!daemon_will_restart(
11221            &mut snapshot_with_restarts(true, 3),
11222            &policy,
11223            now
11224        ));
11225        assert!(!daemon_will_restart(
11226            &mut snapshot_with_restarts(false, 0),
11227            &policy,
11228            now
11229        ));
11230    }
11231
11232    #[test]
11233    fn crash_restart_backoff_escalates_with_in_window_count() {
11234        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11235            .with_max_backoff(Duration::from_secs(30));
11236        let now = ClockInstant::now();
11237        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11238        let schedules = (0..4)
11239            .map(|_| {
11240                state
11241                    .next_crash_restart(&policy, now)
11242                    .expect("the test policy allows four crash restarts")
11243            })
11244            .collect::<Vec<_>>();
11245
11246        assert_eq!(
11247            schedules
11248                .iter()
11249                .map(|schedule| schedule.restart_in_window)
11250                .collect::<Vec<_>>(),
11251            vec![0, 1, 2, 3]
11252        );
11253        assert_eq!(
11254            schedules
11255                .iter()
11256                .map(|schedule| schedule.delay)
11257                .collect::<Vec<_>>(),
11258            vec![
11259                Duration::from_millis(100),
11260                Duration::from_secs(1),
11261                Duration::from_secs(10),
11262                Duration::from_secs(30),
11263            ]
11264        );
11265    }
11266
11267    #[test]
11268    fn crash_restart_backoff_resets_after_ring_clear() {
11269        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11270        let now = ClockInstant::now();
11271        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11272        assert_eq!(
11273            state.next_crash_restart(&policy, now).unwrap().delay,
11274            Duration::from_millis(100)
11275        );
11276        assert_eq!(
11277            state.next_crash_restart(&policy, now).unwrap().delay,
11278            Duration::from_secs(1)
11279        );
11280
11281        state.clear_crash_restarts();
11282        let schedule = state
11283            .next_crash_restart(&policy, now)
11284            .expect("a cleared ring must allow another restart");
11285        assert_eq!(schedule.restart_in_window, 0);
11286        assert_eq!(schedule.delay, Duration::from_millis(100));
11287    }
11288
11289    #[test]
11290    fn crash_restart_backoff_ignores_aged_restarts() {
11291        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11292        let now = ClockInstant::now();
11293        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11294        state
11295            .next_crash_restart(&policy, now)
11296            .expect("the first restart is allowed");
11297        state
11298            .next_crash_restart(&policy, now)
11299            .expect("the second restart is allowed");
11300        state.crash_restarts[0] = now
11301            .checked_sub(policy.window + Duration::from_secs(1))
11302            .expect("the fake clock can age a restart past the window");
11303
11304        let schedule = state
11305            .next_crash_restart(&policy, now)
11306            .expect("an aged restart must release its slot");
11307        assert_eq!(schedule.restart_in_window, 1);
11308        assert_eq!(schedule.delay, Duration::from_secs(1));
11309        assert_eq!(state.crash_restarts.len(), 2);
11310    }
11311
11312    /// The budget is a rate: the same three spent restarts refuse a respawn
11313    /// while they are recent and allow one once they have aged past the window.
11314    /// Nothing about the module changed in between, which is the whole point.
11315    #[test]
11316    fn a_budget_spent_before_the_window_no_longer_refuses() {
11317        let policy = RestartPolicy::new(3, Duration::ZERO);
11318        let mut state = snapshot_with_restarts(true, 3);
11319        let now = ClockInstant::now();
11320        assert!(!daemon_will_restart(&mut state, &policy, now));
11321
11322        assert!(daemon_will_restart(
11323            &mut state,
11324            &policy,
11325            now + policy.window + Duration::from_secs(1)
11326        ));
11327        assert!(
11328            state.crash_restarts.is_empty(),
11329            "reading the budget must drop the instants that left the window"
11330        );
11331    }
11332
11333    fn module_with_recovery_snapshot(
11334        state: ModuleState,
11335        enabled: bool,
11336        restart_count: u32,
11337    ) -> SupervisedModule {
11338        let registry = Arc::new(Registry::default());
11339        let supervisor =
11340            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11341        let runtime = supervisor.runtime_config();
11342        let spec = ModuleSpec {
11343            module_id: "recovery-snapshot".to_string(),
11344            program: fake_aft_stub_path(),
11345            args: Vec::new(),
11346            env: Vec::new(),
11347            reserved: false,
11348            reserved_prefixes: Vec::new(),
11349            protocol: ModuleProtocol::Subc,
11350            overlap: Default::default(),
11351        };
11352        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11353        seed_crash_restarts(&mut snapshot, restart_count);
11354        // These tests read synthetic snapshots. A real child and monitor would
11355        // race those reads by replacing the requested state during startup.
11356        let (commands, _rx) = mpsc::channel(4);
11357        SupervisedModule {
11358            inner: Arc::new(SupervisedModuleInner {
11359                module_id: spec.module_id.clone(),
11360                registry,
11361                snapshot: Arc::new(Mutex::new(snapshot)),
11362                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11363                    spec,
11364                    health: runtime.health,
11365                })),
11366                stderr_ring: runtime.stderr_ring,
11367                terminal_ring: runtime.terminal_ring,
11368                commands,
11369                monitor: Mutex::new(None),
11370                restart_policy: runtime.restart_policy,
11371                effective_drain_timeout: runtime.effective_drain_timeout,
11372                provenance_probe: supervisor.provenance_probe.clone(),
11373            }),
11374        }
11375    }
11376
11377    #[cfg(target_os = "linux")]
11378    #[tokio::test]
11379    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11380        let supervisor =
11381            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11382                .with_cgroup_placement(None);
11383        let result = supervisor.spawn(ModuleSpec {
11384            module_id: "no-cgroup-placement".to_string(),
11385            program: fake_aft_stub_path(),
11386            args: Vec::new(),
11387            env: Vec::new(),
11388            reserved: false,
11389            reserved_prefixes: Vec::new(),
11390            protocol: ModuleProtocol::Subc,
11391            overlap: Default::default(),
11392        });
11393
11394        assert!(
11395            result.is_ok(),
11396            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11397        );
11398    }
11399
11400    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11401    async fn undecided_snapshot_uses_shared_restart_predicate() {
11402        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11403            .will_recover_after_connection_loss()
11404            .unwrap());
11405        assert!(
11406            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11407                .will_recover_after_connection_loss()
11408                .unwrap()
11409        );
11410    }
11411
11412    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11413    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11414        assert!(
11415            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11416                .will_recover_after_connection_loss()
11417                .unwrap()
11418        );
11419    }
11420
11421    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11422    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11423        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11424            .will_recover_after_connection_loss()
11425            .unwrap());
11426        assert!(
11427            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11428                .will_recover_after_connection_loss()
11429                .unwrap()
11430        );
11431    }
11432
11433    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11434    async fn warming_snapshot_is_limited_to_startup_phases() {
11435        for state in [
11436            ModuleState::Starting,
11437            ModuleState::Running,
11438            ModuleState::Restarting,
11439        ] {
11440            assert!(
11441                module_with_recovery_snapshot(state, true, 0)
11442                    .is_warming()
11443                    .unwrap(),
11444                "{state:?} should be warming"
11445            );
11446        }
11447        for state in [
11448            ModuleState::Unresponsive,
11449            ModuleState::Draining,
11450            ModuleState::Stopped,
11451            ModuleState::Failed,
11452            ModuleState::Disabled,
11453        ] {
11454            assert!(
11455                !module_with_recovery_snapshot(state, true, 0)
11456                    .is_warming()
11457                    .unwrap(),
11458                "{state:?} should not be warming"
11459            );
11460        }
11461    }
11462
11463    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11464    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11465        let registry = Arc::new(Registry::default());
11466        let supervisor =
11467            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11468        let module = supervisor
11469            .spawn(ModuleSpec {
11470                module_id: "terminal-history".to_string(),
11471                program: fake_aft_stub_path(),
11472                args: Vec::new(),
11473                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11474                reserved: false,
11475                reserved_prefixes: Vec::new(),
11476                protocol: ModuleProtocol::Subc,
11477                overlap: Default::default(),
11478            })
11479            .unwrap();
11480
11481        let deadline = Instant::now() + Duration::from_secs(5);
11482        loop {
11483            let history = module.terminal_history();
11484            if history.entries.len() == 2 {
11485                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11486                assert_eq!(history.dropped, 0);
11487                assert_eq!(
11488                    history
11489                        .entries
11490                        .iter()
11491                        .map(|entry| entry.exit_code)
11492                        .collect::<Vec<_>>(),
11493                    vec![Some(23), Some(23)]
11494                );
11495                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11496                return;
11497            }
11498            assert!(
11499                Instant::now() < deadline,
11500                "module did not retain two terminal exits: {history:?}"
11501            );
11502            sleep(Duration::from_millis(10)).await;
11503        }
11504    }
11505
11506    /// A disable issued while a crash respawn is still backing off must preempt
11507    /// that respawn: the operator's stop wins, the disable must not queue behind
11508    /// the backoff, and the module must never come back up afterwards.
11509    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11510    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11511        let backoff = Duration::from_secs(2);
11512        let supervisor = Supervisor::new_for_test(
11513            Arc::new(Registry::default()),
11514            RestartPolicy::new(10, backoff),
11515        );
11516        let module = supervisor
11517            .spawn(ModuleSpec {
11518                module_id: "disable-during-backoff".to_string(),
11519                program: fake_aft_stub_path(),
11520                args: Vec::new(),
11521                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11522                reserved: false,
11523                reserved_prefixes: Vec::new(),
11524                protocol: ModuleProtocol::Subc,
11525                overlap: Default::default(),
11526            })
11527            .unwrap();
11528
11529        // Wait for the first crash to put the module into its backoff window.
11530        let deadline = Instant::now() + Duration::from_secs(5);
11531        loop {
11532            if module.status().unwrap().state == ModuleState::Restarting {
11533                break;
11534            }
11535            assert!(
11536                Instant::now() < deadline,
11537                "module never entered the crash backoff"
11538            );
11539            sleep(Duration::from_millis(10)).await;
11540        }
11541
11542        let started = Instant::now();
11543        module.set_enabled(false).await.unwrap();
11544        let waited = started.elapsed();
11545
11546        assert!(
11547            waited < backoff / 2,
11548            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11549        );
11550        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11551
11552        // Outlast the backoff: the respawn it was counting down to must never run.
11553        sleep(backoff + Duration::from_millis(500)).await;
11554        let status = module.status().unwrap();
11555        assert_eq!(status.state, ModuleState::Disabled);
11556        assert_eq!(
11557            status.spawn_generation, 1,
11558            "module respawned after the operator disabled it"
11559        );
11560    }
11561
11562    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
11563    /// the shape of nats-server, the program this rule exists for.
11564    #[cfg(unix)]
11565    fn protocol_none_sigterm_exits_clean_spec(
11566        module_id: &str,
11567        dir: &std::path::Path,
11568    ) -> (ModuleSpec, PathBuf, PathBuf) {
11569        let ready = dir.join("ready");
11570        let marker = dir.join("sigterm");
11571        let spec = ModuleSpec {
11572            module_id: module_id.to_string(),
11573            program: fake_aft_stub_path(),
11574            args: Vec::new(),
11575            env: vec![
11576                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11577                (
11578                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11579                    marker.display().to_string(),
11580                ),
11581                (
11582                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11583                    ready.display().to_string(),
11584                ),
11585            ],
11586            reserved: false,
11587            reserved_prefixes: Vec::new(),
11588            protocol: ModuleProtocol::None,
11589            overlap: Default::default(),
11590        };
11591        (spec, ready, marker)
11592    }
11593
11594    /// Wait for a file the child writes, so a signal is never sent before the
11595    /// child's SIGTERM handler is installed (the default disposition would
11596    /// kill it by signal and the exit would not be clean).
11597    #[cfg(unix)]
11598    async fn wait_for_file(path: &std::path::Path) {
11599        let deadline = Instant::now() + Duration::from_secs(10);
11600        while !path.exists() {
11601            assert!(
11602                Instant::now() < deadline,
11603                "{} never appeared",
11604                path.display()
11605            );
11606            sleep(Duration::from_millis(10)).await;
11607        }
11608    }
11609
11610    /// A protocol-none module that exits 0 because something OUTSIDE the
11611    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
11612    /// the crash-path disposition rather than `stopped`.
11613    #[cfg(unix)]
11614    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11615    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11616        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11617        let (spec, ready, marker) =
11618            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11619        let supervisor = Supervisor::new_for_test(
11620            Arc::new(Registry::default()),
11621            RestartPolicy::new(3, Duration::ZERO),
11622        );
11623        let module = supervisor.spawn(spec).unwrap();
11624        wait_for_file(&ready).await;
11625        // The ready file proves the child installed its SIGTERM handler, not
11626        // that the supervisor has processed the privacy trampoline's exec
11627        // acknowledgement. On macOS status withholds the pid until then.
11628        let deadline = Instant::now() + Duration::from_secs(10);
11629        let first_pid = loop {
11630            let status = module.status().unwrap();
11631            if status.state == ModuleState::Running {
11632                if let Some(pid) = status.pid {
11633                    break pid;
11634                }
11635            }
11636            assert!(
11637                Instant::now() < deadline,
11638                "a running module must report its pid after exec confirmation: {status:?}"
11639            );
11640            sleep(Duration::from_millis(10)).await;
11641        };
11642
11643        rustix::process::kill_process(
11644            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11645            rustix::process::Signal::TERM,
11646        )
11647        .unwrap();
11648
11649        let deadline = Instant::now() + Duration::from_secs(10);
11650        let respawned = loop {
11651            let status = module.status().unwrap();
11652            if status.state == ModuleState::Running
11653                && status.pid.is_some_and(|pid| pid != first_pid)
11654            {
11655                break status;
11656            }
11657            assert!(
11658                Instant::now() < deadline,
11659                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11660            );
11661            sleep(Duration::from_millis(10)).await;
11662        };
11663        assert_eq!(respawned.spawn_generation, 2);
11664        assert!(
11665            marker.exists(),
11666            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11667        );
11668
11669        let history = module.terminal_history();
11670        assert_eq!(history.entries.len(), 1, "{history:?}");
11671        let entry = &history.entries[0];
11672        assert_eq!(entry.exit_code, Some(0));
11673        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11674        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11675
11676        module.stop().await.unwrap();
11677    }
11678
11679    /// Repeated unrequested clean exits of a protocol-none module spend the
11680    /// restart budget exactly as crashes do, and the module ends `failed` with
11681    /// the budget named.
11682    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11683    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11684        let supervisor = Supervisor::new_for_test(
11685            Arc::new(Registry::default()),
11686            RestartPolicy::new(1, Duration::ZERO),
11687        );
11688        let module = supervisor
11689            .spawn(ModuleSpec {
11690                module_id: "none-clean-exit-budget".to_string(),
11691                program: fake_aft_stub_path(),
11692                args: Vec::new(),
11693                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11694                reserved: false,
11695                reserved_prefixes: Vec::new(),
11696                protocol: ModuleProtocol::None,
11697                overlap: Default::default(),
11698            })
11699            .unwrap();
11700
11701        // Failed follows the terminal write; this deadline only bounds a hang,
11702        // not an assumed duration for the two launches or their exit recording.
11703        let deadline = Instant::now() + Duration::from_secs(10);
11704        loop {
11705            let status = module.status().unwrap();
11706            if status.state == ModuleState::Failed {
11707                break;
11708            }
11709            assert!(
11710                Instant::now() < deadline,
11711                "module never exhausted its budget: {status:?} {:?}",
11712                module.terminal_history()
11713            );
11714            sleep(Duration::from_millis(10)).await;
11715        }
11716        let history = module.terminal_history();
11717        assert_eq!(
11718            history
11719                .entries
11720                .iter()
11721                .map(|entry| (entry.exit_code, entry.disposition.clone()))
11722                .collect::<Vec<_>>(),
11723            vec![
11724                (Some(0), TerminalDisposition::Restarting),
11725                (Some(0), TerminalDisposition::Failed),
11726            ]
11727        );
11728        let detail = history.entries[1]
11729            .disposition_detail
11730            .as_deref()
11731            .expect("a budget failure names the budget");
11732        assert!(detail.contains("max_restarts=1"), "{detail}");
11733        assert_eq!(module.status().unwrap().spawn_generation, 2);
11734    }
11735
11736    #[test]
11737    fn restart_budget_failure_is_published_after_its_terminal_record() {
11738        use std::io::{Read, Write};
11739        let supervisor = Supervisor::new_for_test(
11740            Arc::new(Registry::default()),
11741            RestartPolicy::new(0, Duration::ZERO),
11742        );
11743        let runtime = supervisor.runtime_config();
11744        let spec = ModuleSpec {
11745            protocol: ModuleProtocol::None,
11746            ..windowed_crash_spec("budget-publication-order")
11747        };
11748        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11749            ModuleState::Running,
11750            true,
11751        )));
11752        lock_snapshot(&snapshot).unwrap().process_alive = true;
11753        let reaped_snapshot = snapshot.clone();
11754        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11755        let history = runtime.terminal_ring.clone();
11756        let reap = std::thread::spawn(move || {
11757            tokio::runtime::Builder::new_current_thread()
11758                .enable_all()
11759                .build()
11760                .unwrap()
11761                .block_on(on_child_exit(
11762                    &spec,
11763                    runtime.restart_policy,
11764                    &supervisor.registry,
11765                    &reaped_snapshot,
11766                    &runtime.terminal_ring,
11767                    &runtime.spawn_events,
11768                    &runtime.child_roster,
11769                    ExitReport {
11770                        kind: ExitKind::Clean,
11771                        code: Some(0),
11772                        signal: None,
11773                        at_ms: 1,
11774                    },
11775                ))
11776        });
11777        let mut reached = [0];
11778        barrier.read_exact(&mut reached).unwrap();
11779        assert_eq!(&reached, b"R");
11780        let (before_state, before_alive) = {
11781            let state = lock_snapshot(&snapshot).unwrap();
11782            (state.state, state.process_alive)
11783        };
11784        let before_history = history.lock().unwrap().snapshot();
11785        barrier.write_all(b"X").unwrap();
11786        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11787        assert!(before_history.entries.is_empty());
11788        assert_ne!(
11789            before_state,
11790            ModuleState::Failed,
11791            "Failed was visible before its terminal record could be written"
11792        );
11793        assert!(before_alive);
11794        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11795        let history = history.lock().unwrap().snapshot();
11796        assert_eq!(history.entries.len(), 1);
11797        assert_eq!(history.entries[0].exit_code, Some(0));
11798        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11799    }
11800
11801    #[test]
11802    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11803        use std::io::{Read, Write};
11804
11805        let supervisor = Supervisor::new_for_test(
11806            Arc::new(Registry::default()),
11807            RestartPolicy::new(3, Duration::ZERO),
11808        );
11809        let mut runtime = supervisor.runtime_config();
11810        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11811        let snapshot = module.inner.snapshot.clone();
11812        lock_snapshot(&snapshot).unwrap().process_alive = true;
11813        runtime.terminal_ring = module.inner.terminal_ring.clone();
11814        let spec = windowed_crash_spec(module.module_id());
11815        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11816        let reaped_snapshot = snapshot.clone();
11817        let reap = std::thread::spawn(move || {
11818            tokio::runtime::Builder::new_current_thread()
11819                .enable_all()
11820                .build()
11821                .unwrap()
11822                .block_on(on_child_exit(
11823                    &spec,
11824                    runtime.restart_policy,
11825                    &supervisor.registry,
11826                    &reaped_snapshot,
11827                    &runtime.terminal_ring,
11828                    &runtime.spawn_events,
11829                    &runtime.child_roster,
11830                    ExitReport {
11831                        kind: ExitKind::Crash,
11832                        code: Some(2),
11833                        signal: None,
11834                        at_ms: 1,
11835                    },
11836                ))
11837        });
11838
11839        // Hold the crash handler between its budget decision and publication.
11840        // Connection cleanup must still see the recovery that this crash chose,
11841        // even when that recovery consumes the last available restart slot.
11842        let mut reached = [0];
11843        barrier.read_exact(&mut reached).unwrap();
11844        assert_eq!(&reached, b"R");
11845        let before_status = module.status().unwrap();
11846        let before_history = module.terminal_history();
11847        let before_recovers = module.will_recover_after_connection_loss().unwrap();
11848        barrier.write_all(b"X").unwrap();
11849        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11850
11851        assert_eq!(before_status.state, ModuleState::Running);
11852        assert!(before_status.process_alive);
11853        assert_eq!(before_status.restart_count, 3);
11854        assert!(before_history.entries.is_empty());
11855        assert!(
11856            before_recovers,
11857            "connection cleanup would report a terminal close for a scheduled crash restart"
11858        );
11859        let restarting = module.status().unwrap();
11860        assert_eq!(restarting.state, ModuleState::Restarting);
11861        assert_eq!(restarting.restart_count, 3);
11862        assert!(module.will_recover_after_connection_loss().unwrap());
11863        let history = module.terminal_history();
11864        assert_eq!(history.entries.len(), 1);
11865        assert_eq!(
11866            history.entries[0].disposition,
11867            TerminalDisposition::Restarting
11868        );
11869
11870        // Once the replacement is running, its next crash must be judged
11871        // against the exhausted budget rather than the previous recovery.
11872        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11873            state.state = ModuleState::Running;
11874            state.respawn_pending = false;
11875        })
11876        .unwrap();
11877        assert!(!module.will_recover_after_connection_loss().unwrap());
11878        update_snapshot(&snapshot, Some(module.module_id()), |state| {
11879            state.enabled = false;
11880            state.respawn_pending = true;
11881        })
11882        .unwrap();
11883        assert!(!module.will_recover_after_connection_loss().unwrap());
11884    }
11885
11886    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11887        use std::net::{TcpListener, TcpStream};
11888        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11889        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11890        let (barrier, _) = listener.accept().unwrap();
11891        for stream in [&writer, &barrier] {
11892            stream
11893                .set_read_timeout(Some(Duration::from_secs(10)))
11894                .unwrap();
11895            stream
11896                .set_write_timeout(Some(Duration::from_secs(10)))
11897                .unwrap();
11898        }
11899        *events.1.lock().unwrap() = Some(writer);
11900        barrier
11901    }
11902
11903    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11904        use std::io::{Read, Write};
11905        let barrier = events.1.lock().unwrap().take();
11906        if let Some(mut barrier) = barrier {
11907            barrier.write_all(b"R").unwrap();
11908            let mut release = [0];
11909            barrier.read_exact(&mut release).unwrap();
11910            assert_eq!(&release, b"X");
11911        }
11912    }
11913
11914    #[test]
11915    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11916        use std::io::{Read, Write};
11917
11918        let supervisor = Supervisor::new_for_test(
11919            Arc::new(Registry::default()),
11920            RestartPolicy::new(0, Duration::ZERO),
11921        );
11922        let mut runtime = supervisor.runtime_config();
11923        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11924        let snapshot = module.inner.snapshot.clone();
11925        runtime.terminal_ring = module.inner.terminal_ring.clone();
11926        let spec = ModuleSpec {
11927            module_id: module.module_id().into(),
11928            program: fake_aft_stub_path(),
11929            protocol: ModuleProtocol::None,
11930            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11931            ..windowed_crash_spec(module.module_id())
11932        };
11933        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11934        let reap = std::thread::spawn(move || {
11935            tokio::runtime::Builder::new_current_thread()
11936                .enable_all()
11937                .build()
11938                .unwrap()
11939                .block_on(async {
11940                    let mut child =
11941                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11942                    health_restart_child(
11943                        &spec,
11944                        &runtime,
11945                        &supervisor.registry,
11946                        &supervisor.process_liveness,
11947                        &snapshot,
11948                        &mut child,
11949                        SupervisorHealthStatus::Failing,
11950                        Some("health probe did not answer"),
11951                        1,
11952                    )
11953                    .await
11954                    .unwrap();
11955                    assert!(child.is_none());
11956                });
11957        });
11958        // The loopback barrier acknowledges entry into terminal recording by
11959        // health_restart_child's drain, then waits for the test's release byte.
11960        // Status and history are observed while that writer is held, not after
11961        // an elapsed-time guess about when the health action has completed.
11962        let mut reached = [0];
11963        barrier.read_exact(&mut reached).unwrap();
11964        assert_eq!(&reached, b"R");
11965        let before_status = module.status().unwrap();
11966        let before_history = module.terminal_history();
11967        barrier.write_all(b"X").unwrap();
11968        reap.join().unwrap();
11969        assert!(before_history.entries.is_empty());
11970        assert_ne!(
11971            before_status.state,
11972            ModuleState::Failed,
11973            "Failed was visible before its terminal record could be written"
11974        );
11975        assert!(
11976            before_status.process_alive,
11977            "process death was visible before its terminal record could be written"
11978        );
11979        let failed = module.status().unwrap();
11980        assert_eq!(failed.state, ModuleState::Failed);
11981        assert!(!failed.process_alive);
11982        let history = module.terminal_history();
11983        assert_eq!(history.entries.len(), 1);
11984        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11985        assert_eq!(
11986            history.entries[0].disposition_detail.as_deref(),
11987            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11988        );
11989    }
11990
11991    #[test]
11992    fn observed_exits_publish_process_death_after_terminal_history() {
11993        use std::io::{Read, Write};
11994        for (kind, enabled, shutdown, wait_error, expected_state) in [
11995            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11996            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11997            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11998            (
11999                ExitKind::DeliberateSeverance,
12000                true,
12001                false,
12002                false,
12003                ModuleState::Restarting,
12004            ),
12005            #[cfg(unix)]
12006            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12007            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12008        ] {
12009            let supervisor = Supervisor::new_for_test(
12010                Arc::new(Registry::default()),
12011                RestartPolicy::new(1, Duration::ZERO),
12012            );
12013            let runtime = supervisor.runtime_config();
12014            #[cfg(unix)]
12015            if shutdown {
12016                runtime.child_roster.close();
12017            }
12018            #[cfg(not(unix))]
12019            let _ = shutdown;
12020            let spec = windowed_crash_spec("exit-publication-order");
12021            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12022                ModuleState::Running,
12023                enabled,
12024            )));
12025            lock_snapshot(&snapshot).unwrap().process_alive = true;
12026            let reaped_snapshot = snapshot.clone();
12027            let history = runtime.terminal_ring.clone();
12028            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12029            let reap = std::thread::spawn(move || {
12030                if wait_error {
12031                    super::fail_after_wait_error(
12032                        &spec.module_id,
12033                        &reaped_snapshot,
12034                        &runtime.terminal_ring,
12035                        &runtime.spawn_events,
12036                        "wait failed".into(),
12037                    );
12038                } else {
12039                    tokio::runtime::Builder::new_current_thread()
12040                        .enable_all()
12041                        .build()
12042                        .unwrap()
12043                        .block_on(on_child_exit(
12044                            &spec,
12045                            runtime.restart_policy,
12046                            &supervisor.registry,
12047                            &reaped_snapshot,
12048                            &runtime.terminal_ring,
12049                            &runtime.spawn_events,
12050                            &runtime.child_roster,
12051                            ExitReport {
12052                                kind,
12053                                code: Some(0),
12054                                signal: None,
12055                                at_ms: 1,
12056                            },
12057                        ));
12058                }
12059            });
12060            let mut reached = [0];
12061            barrier.read_exact(&mut reached).unwrap();
12062            assert_eq!(&reached, b"R");
12063            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12064            let before_history = history.lock().unwrap().snapshot();
12065            barrier.write_all(b"X").unwrap();
12066            reap.join().unwrap();
12067            assert!(
12068                before_alive,
12069                "{kind:?}: death was published before recording"
12070            );
12071            assert!(before_history.entries.is_empty());
12072            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12073            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12074            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12075        }
12076    }
12077
12078    #[tokio::test]
12079    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12080        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12081        let path = dir.join("terminals.jsonl");
12082        std::fs::create_dir(&path).unwrap();
12083        let supervisor = Supervisor::new_for_test(
12084            Arc::new(Registry::default()),
12085            RestartPolicy::new(0, Duration::ZERO),
12086        )
12087        .with_terminal_journal(path, "health-budget-journal-failure".into());
12088        let runtime = supervisor.runtime_config();
12089        let spec = ModuleSpec {
12090            program: fake_aft_stub_path(),
12091            protocol: ModuleProtocol::None,
12092            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12093            ..windowed_crash_spec("health-budget-journal-failure")
12094        };
12095        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12096        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12097        health_restart_child(
12098            &spec,
12099            &runtime,
12100            &supervisor.registry,
12101            &supervisor.process_liveness,
12102            &snapshot,
12103            &mut child,
12104            SupervisorHealthStatus::Failing,
12105            None,
12106            1,
12107        )
12108        .await
12109        .unwrap();
12110        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12111        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12112        let history = runtime
12113            .terminal_ring
12114            .lock()
12115            .unwrap()
12116            .durable_history(&spec.module_id);
12117        assert!(history.journal_write_failures > 0);
12118        assert_eq!(history.entries.len(), 1);
12119        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12120    }
12121
12122    #[test]
12123    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12124        let supervisor = Supervisor::new_for_test(
12125            Arc::new(Registry::default()),
12126            RestartPolicy::new(0, Duration::ZERO),
12127        );
12128        let runtime = supervisor.runtime_config();
12129        let spec = ModuleSpec {
12130            program: fake_aft_stub_path(),
12131            protocol: ModuleProtocol::None,
12132            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12133            ..windowed_crash_spec("health-budget-recording-panic")
12134        };
12135        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12136        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12137            tokio::runtime::Builder::new_current_thread()
12138                .enable_all()
12139                .build()
12140                .unwrap()
12141                .block_on(async {
12142                    let mut child =
12143                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12144                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12145                    health_restart_child(
12146                        &spec,
12147                        &runtime,
12148                        &supervisor.registry,
12149                        &supervisor.process_liveness,
12150                        &snapshot,
12151                        &mut child,
12152                        SupervisorHealthStatus::Failing,
12153                        None,
12154                        1,
12155                    )
12156                    .await
12157                    .unwrap();
12158                });
12159        }));
12160        let panic = result.expect_err("recording must still unwind");
12161        assert_eq!(
12162            panic.downcast_ref::<String>().map(String::as_str),
12163            Some("spawn event sequence exhausted")
12164        );
12165        let state = lock_snapshot(&snapshot).unwrap();
12166        assert_eq!(state.state, ModuleState::Failed);
12167        assert!(!state.process_alive);
12168        assert!(state.last_exit.is_some());
12169    }
12170
12171    #[tokio::test]
12172    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12173        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12174        let path = dir.join("terminals.jsonl");
12175        std::fs::create_dir(&path).unwrap();
12176        let supervisor = Supervisor::new_for_test(
12177            Arc::new(Registry::default()),
12178            RestartPolicy::new(0, Duration::ZERO),
12179        )
12180        .with_terminal_journal(path, "budget-journal-failure".into());
12181        let runtime = supervisor.runtime_config();
12182        let spec = windowed_crash_spec("budget-journal-failure");
12183        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12184        assert!(matches!(
12185            on_child_exit(
12186                &spec,
12187                runtime.restart_policy,
12188                &supervisor.registry,
12189                &snapshot,
12190                &runtime.terminal_ring,
12191                &runtime.spawn_events,
12192                &runtime.child_roster,
12193                crash_exit_report(1),
12194            )
12195            .await,
12196            NextAction::Stop { .. }
12197        ));
12198        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12199        let history = runtime
12200            .terminal_ring
12201            .lock()
12202            .unwrap()
12203            .durable_history(&spec.module_id);
12204        assert!(history.journal_write_failures > 0);
12205        assert_eq!(history.entries.len(), 1);
12206        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12207    }
12208
12209    #[test]
12210    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12211        let supervisor = Supervisor::new_for_test(
12212            Arc::new(Registry::default()),
12213            RestartPolicy::new(0, Duration::ZERO),
12214        );
12215        let runtime = supervisor.runtime_config();
12216        let spec = windowed_crash_spec("budget-recording-panic");
12217        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12218        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12219        // Exhausting the event sequence makes emit_exited panic before the
12220        // terminal write, exercising publication on the recording unwind.
12221        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12222        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12223            tokio::runtime::Builder::new_current_thread()
12224                .enable_all()
12225                .build()
12226                .unwrap()
12227                .block_on(on_child_exit(
12228                    &spec,
12229                    runtime.restart_policy,
12230                    &supervisor.registry,
12231                    &snapshot,
12232                    &runtime.terminal_ring,
12233                    &runtime.spawn_events,
12234                    &runtime.child_roster,
12235                    crash_exit_report(1),
12236                ))
12237        }));
12238        let panic = result.err().expect("recording must still unwind");
12239        assert_eq!(
12240            panic.downcast_ref::<String>().map(String::as_str),
12241            Some("spawn event sequence exhausted")
12242        );
12243        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12244    }
12245
12246    /// A stop the supervisor itself requests still stops a protocol-none
12247    /// module, even though the child answers the SIGTERM with exit 0.
12248    #[cfg(unix)]
12249    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12250    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12251        for disable in [false, true] {
12252            let label = if disable {
12253                "none-requested-disable"
12254            } else {
12255                "none-requested-stop"
12256            };
12257            let dir = cortexkit_test_support::ScratchDir::new(label);
12258            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12259            let supervisor = Supervisor::new_for_test(
12260                Arc::new(Registry::default()),
12261                RestartPolicy::new(3, Duration::ZERO),
12262            );
12263            let module = supervisor.spawn(spec).unwrap();
12264            wait_for_file(&ready).await;
12265
12266            if disable {
12267                module.set_enabled(false).await.unwrap();
12268            } else {
12269                module.stop().await.unwrap();
12270            }
12271            assert!(
12272                marker.exists(),
12273                "{label}: the child must have left through its SIGTERM handler with exit 0"
12274            );
12275
12276            // Long enough for a zero-backoff respawn to have happened if the
12277            // exit had been treated as a crash.
12278            sleep(Duration::from_millis(500)).await;
12279            let status = module.status().unwrap();
12280            let expected = if disable {
12281                ModuleState::Disabled
12282            } else {
12283                ModuleState::Stopped
12284            };
12285            assert_eq!(status.state, expected, "{label}");
12286            assert_eq!(
12287                status.spawn_generation, 1,
12288                "{label}: respawned after a requested stop"
12289            );
12290            let history = module.terminal_history();
12291            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12292            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12293            assert_ne!(
12294                history.entries[0].disposition,
12295                TerminalDisposition::Restarting,
12296                "{label}"
12297            );
12298        }
12299    }
12300
12301    /// A subc-wire module that exits 0 on its own is still a stop: the
12302    /// protocol-none rule must not reach it.
12303    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12304    async fn subc_wire_clean_exit_is_still_a_stop() {
12305        let supervisor = Supervisor::new_for_test(
12306            Arc::new(Registry::default()),
12307            RestartPolicy::new(3, Duration::ZERO),
12308        );
12309        let module = supervisor
12310            .spawn(ModuleSpec {
12311                module_id: "wire-clean-exit".to_string(),
12312                program: fake_aft_stub_path(),
12313                args: Vec::new(),
12314                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12315                reserved: false,
12316                reserved_prefixes: Vec::new(),
12317                protocol: ModuleProtocol::Subc,
12318                overlap: Default::default(),
12319            })
12320            .unwrap();
12321
12322        let deadline = Instant::now() + Duration::from_secs(10);
12323        while module.terminal_history().entries.is_empty() {
12324            assert!(Instant::now() < deadline, "module never exited");
12325            sleep(Duration::from_millis(10)).await;
12326        }
12327        // Long enough for a zero-backoff respawn to have happened.
12328        sleep(Duration::from_millis(500)).await;
12329        let status = module.status().unwrap();
12330        assert_eq!(status.state, ModuleState::Stopped);
12331        assert_eq!(status.spawn_generation, 1);
12332        let history = module.terminal_history();
12333        assert_eq!(history.entries.len(), 1, "{history:?}");
12334        assert_eq!(history.entries[0].exit_code, Some(0));
12335        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12336    }
12337
12338    #[cfg(unix)]
12339    #[tokio::test]
12340    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12341        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12342        let record = dir.join("live-children.json");
12343        let supervisor = Supervisor::new_for_test(
12344            Arc::new(Registry::default()),
12345            RestartPolicy::new(0, Duration::ZERO),
12346        );
12347        let mut runtime = supervisor.runtime_config();
12348        runtime.child_roster.record_to(record.clone());
12349        let gate = Arc::new(super::ReloadExitRecordGate::default());
12350        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12351        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12352        let spec = ModuleSpec {
12353            module_id: "reload-exit-roster".into(),
12354            program: fake_aft_stub_path(),
12355            args: Vec::new(),
12356            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12357            reserved: false,
12358            reserved_prefixes: Vec::new(),
12359            protocol: ModuleProtocol::Subc,
12360            overlap: Default::default(),
12361        };
12362        let mut child = None;
12363        let reload = super::finish_reload_child(
12364            &spec,
12365            &runtime,
12366            &supervisor.registry,
12367            &supervisor.process_liveness,
12368            &snapshot,
12369            &mut child,
12370        );
12371        tokio::pin!(reload);
12372        tokio::select! {
12373            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12374            _ = gate.reached.notified() => {}
12375        }
12376        assert!(runtime
12377            .terminal_ring
12378            .lock()
12379            .unwrap()
12380            .snapshot()
12381            .entries
12382            .is_empty());
12383        assert_eq!(
12384            crate::live_children::read_record(&record).unwrap().len(),
12385            1,
12386            "shutdown must still wait for the reaped child until its terminal record exists"
12387        );
12388        runtime.child_roster.close();
12389        gate.resume.notify_one();
12390        assert!(reload.await.is_err());
12391        assert!(crate::live_children::read_record(&record)
12392            .unwrap()
12393            .is_empty());
12394        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12395        assert_eq!(history.entries.len(), 1);
12396        assert_eq!(
12397            history.entries[0].disposition,
12398            TerminalDisposition::DaemonShutdown
12399        );
12400    }
12401
12402    /// Each restart-producing arm has its own state transition. Keeping their
12403    /// lifetime count assertions adjacent prevents a later new arm from silently
12404    /// spending budget without recording the historical restart.
12405    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12406    async fn every_restart_increment_path_advances_lifetime_count() {
12407        let supervisor = Supervisor::new_for_test(
12408            Arc::new(Registry::default()),
12409            RestartPolicy::new(1, Duration::ZERO),
12410        );
12411        let runtime = supervisor.runtime_config();
12412        let spec = ModuleSpec {
12413            module_id: "lifetime-increment-path".to_string(),
12414            program: PathBuf::from("/unused/lifetime-increment-path"),
12415            args: Vec::new(),
12416            env: Vec::new(),
12417            reserved: false,
12418            reserved_prefixes: Vec::new(),
12419            protocol: ModuleProtocol::Subc,
12420            overlap: Default::default(),
12421        };
12422
12423        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12424        assert!(matches!(
12425            on_child_exit(
12426                &spec,
12427                runtime.restart_policy,
12428                &supervisor.registry,
12429                &crash_snapshot,
12430                &runtime.terminal_ring,
12431                &runtime.spawn_events,
12432                &runtime.child_roster,
12433                ExitReport {
12434                    kind: ExitKind::Crash,
12435                    code: Some(1),
12436                    signal: None,
12437                    at_ms: 1,
12438                },
12439            )
12440            .await,
12441            NextAction::Restart { schedule: _ }
12442        ));
12443        let (crash_restarts, crash_lifetime) = {
12444            let state = lock_snapshot(&crash_snapshot).unwrap();
12445            (state.crash_restarts.len(), state.lifetime_restarts)
12446        };
12447        assert_eq!(crash_restarts, 1);
12448        assert_eq!(crash_lifetime, 1);
12449
12450        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12451        let mut health_child = None;
12452        assert!(matches!(
12453            health_restart_child(
12454                &spec,
12455                &runtime,
12456                &supervisor.registry,
12457                &supervisor.process_liveness,
12458                &health_snapshot,
12459                &mut health_child,
12460                SupervisorHealthStatus::Failing,
12461                None,
12462                2,
12463            )
12464            .await,
12465            Ok(())
12466        ));
12467        assert!(health_child.is_none());
12468        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12469        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12470        let (health_restarts, health_lifetime) = {
12471            let state = lock_snapshot(&health_snapshot).unwrap();
12472            (state.crash_restarts.len(), state.lifetime_restarts)
12473        };
12474        assert_eq!(health_restarts, 1);
12475        assert_eq!(health_lifetime, 1);
12476
12477        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12478        let mut reload_child = None;
12479        assert!(matches!(
12480            handle_reload_spawn_failure(
12481                &spec,
12482                &runtime,
12483                &supervisor.process_liveness,
12484                &reload_snapshot,
12485                &mut reload_child,
12486                "forced reload spawn failure".to_string(),
12487            )
12488            .await,
12489            Err(SuperviseError::ReloadFailed { .. })
12490        ));
12491        let (reload_restarts, reload_lifetime) = {
12492            let state = lock_snapshot(&reload_snapshot).unwrap();
12493            (state.crash_restarts.len(), state.lifetime_restarts)
12494        };
12495        assert_eq!(reload_restarts, 1);
12496        assert_eq!(reload_lifetime, 1);
12497    }
12498
12499    #[tokio::test]
12500    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12501        let supervisor = Supervisor::new_for_test(
12502            Arc::new(Registry::default()),
12503            RestartPolicy::new(3, Duration::ZERO),
12504        );
12505        let runtime = supervisor.runtime_config();
12506        let spec = ModuleSpec {
12507            module_id: "deliberately-severed".to_string(),
12508            program: PathBuf::from("/unused/deliberately-severed"),
12509            args: Vec::new(),
12510            env: Vec::new(),
12511            reserved: false,
12512            reserved_prefixes: Vec::new(),
12513            protocol: ModuleProtocol::Subc,
12514            overlap: Default::default(),
12515        };
12516        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12517        let process = ProcessIdentity {
12518            pid: 41,
12519            start_time: 101,
12520        };
12521        record_deliberate_severance(&snapshot, process).unwrap();
12522        let exit_report = apply_deliberate_severance_marker(
12523            &snapshot,
12524            Some(process),
12525            ExitReport {
12526                kind: ExitKind::Crash,
12527                code: Some(1),
12528                signal: None,
12529                at_ms: 1,
12530            },
12531        );
12532        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12533
12534        assert!(matches!(
12535            on_child_exit(
12536                &spec,
12537                runtime.restart_policy,
12538                &supervisor.registry,
12539                &snapshot,
12540                &runtime.terminal_ring,
12541                &runtime.spawn_events,
12542                &runtime.child_roster,
12543                exit_report,
12544            )
12545            .await,
12546            NextAction::Restart { schedule: _ }
12547        ));
12548        let state = lock_snapshot(&snapshot).unwrap();
12549        assert_eq!(state.lifetime_restarts, 1);
12550        assert_eq!(state.crash_restarts.len(), 0);
12551    }
12552
12553    #[tokio::test]
12554    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12555        let supervisor = Supervisor::new_for_test(
12556            Arc::new(Registry::default()),
12557            RestartPolicy::new(3, Duration::ZERO),
12558        );
12559        let runtime = supervisor.runtime_config();
12560        let spec = ModuleSpec {
12561            module_id: "genuine-crash".to_string(),
12562            program: PathBuf::from("/unused/genuine-crash"),
12563            args: Vec::new(),
12564            env: Vec::new(),
12565            reserved: false,
12566            reserved_prefixes: Vec::new(),
12567            protocol: ModuleProtocol::Subc,
12568            overlap: Default::default(),
12569        };
12570        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12571
12572        assert!(matches!(
12573            on_child_exit(
12574                &spec,
12575                runtime.restart_policy,
12576                &supervisor.registry,
12577                &snapshot,
12578                &runtime.terminal_ring,
12579                &runtime.spawn_events,
12580                &runtime.child_roster,
12581                ExitReport {
12582                    kind: ExitKind::Crash,
12583                    code: Some(1),
12584                    signal: None,
12585                    at_ms: 1,
12586                },
12587            )
12588            .await,
12589            NextAction::Restart { schedule: _ }
12590        ));
12591        let state = lock_snapshot(&snapshot).unwrap();
12592        assert_eq!(state.lifetime_restarts, 1);
12593        assert_eq!(state.crash_restarts.len(), 1);
12594    }
12595
12596    fn crash_exit_report(at_ms: u64) -> ExitReport {
12597        ExitReport {
12598            kind: ExitKind::Crash,
12599            code: Some(1),
12600            signal: None,
12601            at_ms,
12602        }
12603    }
12604
12605    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12606        ModuleSpec {
12607            module_id: module_id.to_string(),
12608            program: PathBuf::from("/unused").join(module_id),
12609            args: Vec::new(),
12610            env: Vec::new(),
12611            reserved: false,
12612            reserved_prefixes: Vec::new(),
12613            protocol: ModuleProtocol::Subc,
12614            overlap: Default::default(),
12615        }
12616    }
12617
12618    /// A real crash loop still stops. Three crashes with nothing aging out spend
12619    /// a budget of two and the third respawn is refused, and both surfaces an
12620    /// operator has -- the log line and the retained terminal record -- name the
12621    /// window rather than only the cap, because `max_restarts=2` alone is what
12622    /// this budget used to mean.
12623    #[tokio::test]
12624    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12625        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12626        let supervisor = Supervisor::new_for_test(
12627            Arc::new(Registry::default()),
12628            RestartPolicy::new(2, Duration::ZERO),
12629        );
12630        let runtime = supervisor.runtime_config();
12631        let spec = windowed_crash_spec("crash-loop-in-window");
12632        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12633
12634        for attempt in 1..=2 {
12635            assert!(
12636                matches!(
12637                    on_child_exit(
12638                        &spec,
12639                        runtime.restart_policy,
12640                        &supervisor.registry,
12641                        &snapshot,
12642                        &runtime.terminal_ring,
12643                        &runtime.spawn_events,
12644                        &runtime.child_roster,
12645                        crash_exit_report(attempt),
12646                    )
12647                    .await,
12648                    NextAction::Restart { schedule: _ }
12649                ),
12650                "crash {attempt} is inside the budget and must respawn"
12651            );
12652        }
12653
12654        assert!(matches!(
12655            on_child_exit(
12656                &spec,
12657                runtime.restart_policy,
12658                &supervisor.registry,
12659                &snapshot,
12660                &runtime.terminal_ring,
12661                &runtime.spawn_events,
12662                &runtime.child_roster,
12663                crash_exit_report(3),
12664            )
12665            .await,
12666            NextAction::Stop { .. }
12667        ));
12668
12669        {
12670            let state = lock_snapshot(&snapshot).unwrap();
12671            assert_eq!(state.state, ModuleState::Failed);
12672            assert_eq!(state.crash_restarts.len(), 2);
12673            assert_eq!(state.lifetime_restarts, 2);
12674        }
12675
12676        let history = runtime
12677            .terminal_ring
12678            .lock()
12679            .expect("terminal ring is not poisoned")
12680            .snapshot();
12681        let last = history
12682            .entries
12683            .last()
12684            .expect("the refused crash is retained");
12685        assert_eq!(last.disposition, TerminalDisposition::Failed);
12686        assert_eq!(
12687            last.disposition_detail.as_deref(),
12688            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12689        );
12690
12691        let captured = crate::router::test_log::captured_logs(&logs);
12692        assert!(
12693            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12694            "the stop must be logged with its window: {captured}"
12695        );
12696    }
12697
12698    /// The rate, stated as a test: three crashes where the first has aged past
12699    /// the window are two crashes as far as the budget is concerned, so the
12700    /// third respawn is allowed and the ring holds only the two recent ones.
12701    ///
12702    /// This is the case a lifetime counter got wrong -- and the case the daemon
12703    /// now hits routinely, since a module exits non-zero every time its
12704    /// connection to the daemon drops.
12705    #[tokio::test]
12706    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12707        let supervisor = Supervisor::new_for_test(
12708            Arc::new(Registry::default()),
12709            RestartPolicy::new(2, Duration::ZERO),
12710        );
12711        let runtime = supervisor.runtime_config();
12712        let spec = windowed_crash_spec("crash-across-windows");
12713        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12714
12715        for attempt in 1..=2 {
12716            assert!(matches!(
12717                on_child_exit(
12718                    &spec,
12719                    runtime.restart_policy,
12720                    &supervisor.registry,
12721                    &snapshot,
12722                    &runtime.terminal_ring,
12723                    &runtime.spawn_events,
12724                    &runtime.child_roster,
12725                    crash_exit_report(attempt),
12726                )
12727                .await,
12728                NextAction::Restart { schedule: _ }
12729            ));
12730        }
12731
12732        // The oldest crash moves out of the window; nothing else about the
12733        // module changes.
12734        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12735            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12736        })
12737        .unwrap();
12738
12739        assert!(
12740            matches!(
12741                on_child_exit(
12742                    &spec,
12743                    runtime.restart_policy,
12744                    &supervisor.registry,
12745                    &snapshot,
12746                    &runtime.terminal_ring,
12747                    &runtime.spawn_events,
12748                    &runtime.child_roster,
12749                    crash_exit_report(3),
12750                )
12751                .await,
12752                NextAction::Restart { schedule: _ }
12753            ),
12754            "a crash older than the window must not hold a budget slot"
12755        );
12756
12757        let state = lock_snapshot(&snapshot).unwrap();
12758        assert_eq!(state.state, ModuleState::Restarting);
12759        assert_eq!(
12760            state.crash_restarts.len(),
12761            2,
12762            "the aged instant is dropped and the new one takes its place"
12763        );
12764        assert_eq!(
12765            state.lifetime_restarts, 3,
12766            "the ledger counts every restart, including the ones the window forgot"
12767        );
12768    }
12769
12770    /// An operator restart hands the budget back whole, and the ledger keeps
12771    /// counting. Those are different questions -- "how close is this module to
12772    /// being stopped" and "how many times has it been replaced" -- and the
12773    /// operator action answers only the first.
12774    #[tokio::test]
12775    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12776        let supervisor = Supervisor::new_for_test(
12777            Arc::new(Registry::default()),
12778            RestartPolicy::new(2, Duration::ZERO),
12779        );
12780        let runtime = supervisor.runtime_config();
12781        let spec = windowed_crash_spec("operator-cleared-budget");
12782        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12783
12784        for attempt in 1..=2 {
12785            assert!(matches!(
12786                on_child_exit(
12787                    &spec,
12788                    runtime.restart_policy,
12789                    &supervisor.registry,
12790                    &snapshot,
12791                    &runtime.terminal_ring,
12792                    &runtime.spawn_events,
12793                    &runtime.child_roster,
12794                    crash_exit_report(attempt),
12795                )
12796                .await,
12797                NextAction::Restart { schedule: _ }
12798            ));
12799        }
12800
12801        reset_restart_count(&snapshot, &spec.module_id).unwrap();
12802        {
12803            let state = lock_snapshot(&snapshot).unwrap();
12804            assert!(
12805                state.crash_restarts.is_empty(),
12806                "an operator restart returns the full budget"
12807            );
12808            assert_eq!(
12809                state.lifetime_restarts, 2,
12810                "clearing the budget must not unmake the crashes"
12811            );
12812        }
12813
12814        assert!(
12815            matches!(
12816                on_child_exit(
12817                    &spec,
12818                    runtime.restart_policy,
12819                    &supervisor.registry,
12820                    &snapshot,
12821                    &runtime.terminal_ring,
12822                    &runtime.spawn_events,
12823                    &runtime.child_roster,
12824                    crash_exit_report(3),
12825                )
12826                .await,
12827                NextAction::Restart { schedule: _ }
12828            ),
12829            "the cleared budget must be spendable again"
12830        );
12831        let state = lock_snapshot(&snapshot).unwrap();
12832        assert_eq!(state.crash_restarts.len(), 1);
12833        assert_eq!(state.lifetime_restarts, 3);
12834    }
12835
12836    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12837    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12838        let severed = ProcessIdentity {
12839            pid: 41,
12840            start_time: 101,
12841        };
12842        let successor = ProcessIdentity {
12843            pid: 41,
12844            start_time: 202,
12845        };
12846        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12847        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12848            state.pid = Some(successor.pid);
12849            state.process_start_time = Some(successor.start_time);
12850        })
12851        .unwrap();
12852        assert!(!module.record_deliberate_severance(severed).unwrap());
12853
12854        let exit_report = apply_deliberate_severance_marker(
12855            &module.inner.snapshot,
12856            Some(successor),
12857            ExitReport {
12858                kind: ExitKind::Crash,
12859                code: Some(1),
12860                signal: None,
12861                at_ms: 1,
12862            },
12863        );
12864
12865        assert_eq!(exit_report.kind, ExitKind::Crash);
12866    }
12867
12868    #[tokio::test]
12869    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12870        let registry = Registry::default();
12871        let supervisor = Supervisor::new_for_test(
12872            Arc::new(Registry::default()),
12873            RestartPolicy::new(3, Duration::ZERO),
12874        );
12875        let runtime = supervisor.runtime_config();
12876        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12877        let spec = ModuleSpec {
12878            module_id: "drain-deliberate-severance".to_string(),
12879            program: fake_aft_stub_path(),
12880            args: Vec::new(),
12881            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12882            reserved: false,
12883            reserved_prefixes: Vec::new(),
12884            protocol: ModuleProtocol::Subc,
12885            overlap: Default::default(),
12886        };
12887        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12888        let process = ProcessIdentity {
12889            pid: 41,
12890            start_time: 101,
12891        };
12892        child.process_identity = Some(process);
12893        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12894            state.pid = Some(process.pid);
12895            state.process_start_time = Some(process.start_time);
12896        })
12897        .unwrap();
12898        record_deliberate_severance(&snapshot, process).unwrap();
12899
12900        drain_child_to_state(
12901            &spec.module_id,
12902            spec.protocol,
12903            // The child exits on its own; no signal may change the exit this
12904            // test classifies.
12905            StopNotice::SentOverConnection,
12906            &registry,
12907            None,
12908            &snapshot,
12909            &runtime.terminal_ring,
12910            &runtime.spawn_events,
12911            child,
12912            Duration::from_secs(1),
12913            ModuleState::Stopped,
12914            Some(false),
12915        )
12916        .await
12917        .unwrap();
12918
12919        let state = lock_snapshot(&snapshot).unwrap();
12920        assert_eq!(
12921            state.last_exit.as_ref().map(|exit| exit.kind),
12922            Some(ExitKind::DeliberateSeverance)
12923        );
12924        assert_eq!(state.lifetime_restarts, 1);
12925        assert_eq!(state.crash_restarts.len(), 0);
12926        drop(state);
12927        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12928        assert_eq!(
12929            history.entries[0].exit_kind,
12930            subc_control::TerminalExitKind::DeliberateSeverance
12931        );
12932    }
12933
12934    #[tokio::test]
12935    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12936        let registry = Registry::default();
12937        let supervisor = Supervisor::new_for_test(
12938            Arc::new(Registry::default()),
12939            RestartPolicy::new(3, Duration::ZERO),
12940        );
12941        let runtime = supervisor.runtime_config();
12942        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12943        let spec = ModuleSpec {
12944            module_id: "ordinary-drain".to_string(),
12945            program: fake_aft_stub_path(),
12946            args: Vec::new(),
12947            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12948            reserved: false,
12949            reserved_prefixes: Vec::new(),
12950            protocol: ModuleProtocol::Subc,
12951            overlap: Default::default(),
12952        };
12953        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12954
12955        drain_child_to_state(
12956            &spec.module_id,
12957            spec.protocol,
12958            // The child exits on its own; no signal may change the exit this
12959            // test classifies.
12960            StopNotice::SentOverConnection,
12961            &registry,
12962            None,
12963            &snapshot,
12964            &runtime.terminal_ring,
12965            &runtime.spawn_events,
12966            child,
12967            Duration::from_secs(1),
12968            ModuleState::Stopped,
12969            Some(false),
12970        )
12971        .await
12972        .unwrap();
12973
12974        let state = lock_snapshot(&snapshot).unwrap();
12975        assert_eq!(
12976            state.last_exit.as_ref().map(|exit| exit.kind),
12977            Some(ExitKind::Crash)
12978        );
12979        assert_eq!(state.lifetime_restarts, 0);
12980        assert_eq!(state.crash_restarts.len(), 0);
12981    }
12982
12983    #[test]
12984    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12985        // The server's generic fatal-routing branch only knows that the
12986        // connection failed; it does not know that the daemon deliberately
12987        // initiated a process-killing severance. Keep this seam explicit so a
12988        // future connection error path cannot silently reintroduce the stale
12989        // exemption that mislabels a later genuine crash.
12990        assert!(!include_str!("server.rs")
12991            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12992    }
12993
12994    /// The `route.closed` `drained` value must be the quiescence wait's own
12995    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
12996    /// measurement at all and `false` is the one honest constant. This is the exact
12997    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
12998    /// on every return path, including the one that used to return early via `?`
12999    /// with `route.closing` already sent and no `route.closed` ever following.
13000    #[test]
13001    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13002        assert!(drained_after_quiescence_wait(&Ok(true)));
13003        assert!(!drained_after_quiescence_wait(&Ok(false)));
13004        assert!(!drained_after_quiescence_wait(&Err(
13005            SuperviseError::StatePoisoned { module_id: None }
13006        )));
13007    }
13008
13009    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13010    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13011    /// already reaped out-of-band) still leaves a terminal record rather than none
13012    /// at all. Triggering the real `wait()` I/O error from an integration test would
13013    /// need a genuine already-reaped-child race, which is OS-specific and not
13014    /// something this suite attempts elsewhere; this test instead verifies the
13015    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13016    /// the call site itself is verified by inspection to sit in that exact arm.
13017    #[test]
13018    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13019        let ring = Arc::new(Mutex::new(TerminalRing::new(
13020            TerminalRingConfig::default(),
13021            0,
13022        )));
13023        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13024
13025        let snapshot = ring.lock().unwrap().snapshot();
13026        assert_eq!(snapshot.entries.len(), 1);
13027        let entry = &snapshot.entries[0];
13028        assert_eq!(entry.exit_code, None);
13029        assert_eq!(entry.exit_signal, None);
13030        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13031    }
13032
13033    #[test]
13034    fn wait_error_exit_path_preserves_spawn_event_density() {
13035        let feed = super::SpawnEventFeed::default();
13036        feed.configure_incarnation("wait-error-density".to_string());
13037        feed.emit_spawned("wait-error", 41, 1);
13038        let ring = Arc::new(Mutex::new(TerminalRing::new(
13039            TerminalRingConfig::default(),
13040            0,
13041        )));
13042
13043        record_wait_error_terminal("wait-error", &ring, &feed);
13044        feed.emit_spawned("after-wait-error", 42, 2);
13045
13046        let state = feed.0.lock().unwrap();
13047        let sequences = state
13048            .events
13049            .iter()
13050            .map(|event| event.cursor.seq)
13051            .collect::<Vec<_>>();
13052        assert_eq!(sequences, vec![1, 2, 3]);
13053        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13054        assert_eq!(state.events[1].exit_code, None);
13055        assert_eq!(state.events[1].exit_signal, None);
13056    }
13057
13058    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13059    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13060    /// not a clean exit it never actually observed.
13061    #[test]
13062    fn wait_error_exit_report_is_classified_as_a_crash() {
13063        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13064    }
13065}
13066
13067#[cfg(all(test, unix))]
13068mod restart_ordering_tests {
13069    use super::*;
13070    use std::io::Write;
13071    use tokio::io::AsyncReadExt;
13072
13073    #[test]
13074    #[ignore = "child process fixture"]
13075    fn serving_child() {
13076        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13077            return;
13078        };
13079        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13080        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13081        for stream in listener.incoming() {
13082            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13083        }
13084    }
13085
13086    async fn restart_before_publication(requests: usize) {
13087        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13088        let address_file = home.path().join("address");
13089        let spec = ModuleSpec {
13090            module_id: "restart-serving-child".into(),
13091            program: std::env::current_exe().unwrap(),
13092            args: vec![
13093                "--exact".into(),
13094                "supervise::restart_ordering_tests::serving_child".into(),
13095                "--ignored".into(),
13096                "--nocapture".into(),
13097            ],
13098            env: vec![(
13099                "RESTART_TEST_ADDRESS_FILE".into(),
13100                address_file.display().to_string(),
13101            )],
13102            reserved: false,
13103            reserved_prefixes: vec![],
13104            protocol: ModuleProtocol::None,
13105            overlap: Default::default(),
13106        };
13107        let mut supervisor =
13108            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13109        let (spawned_tx, spawned_rx) = oneshot::channel();
13110        let spawned_tx = Mutex::new(Some(spawned_tx));
13111        let (release_tx, release_rx) = std::sync::mpsc::channel();
13112        let release_rx = Mutex::new(release_rx);
13113        // Hold publication, not the child: it must answer a real request before
13114        // the restart enters the queue, even though status still has generation 0.
13115        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13116            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13117            release_rx
13118                .lock()
13119                .unwrap()
13120                .recv_timeout(Duration::from_secs(20))
13121                .unwrap();
13122        })));
13123        let registered = supervisor
13124            .register_configured_with_health(
13125                spec.clone(),
13126                true,
13127                HealthConfig::default(),
13128                None,
13129                RestartPolicy::new(3, Duration::ZERO),
13130            )
13131            .unwrap();
13132        let module = registered.module.clone();
13133        let startup = tokio::task::spawn_blocking(move || {
13134            supervisor.start_registered_configured(registered, None)
13135        });
13136        spawned_rx.await.unwrap();
13137        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13138            let address = loop {
13139                if let Ok(address) = std::fs::read_to_string(&address_file) {
13140                    if !address.is_empty() {
13141                        break address;
13142                    }
13143                }
13144                tokio::time::sleep(Duration::from_millis(1)).await;
13145            };
13146            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13147            let mut reply = String::new();
13148            stream.read_to_string(&mut reply).await.unwrap();
13149            reply.trim().parse::<u32>().unwrap()
13150        })
13151        .await
13152        .expect("child must serve before generation publication");
13153        assert_eq!(
13154            lock_snapshot(&module.inner.snapshot)
13155                .unwrap()
13156                .spawn_generation,
13157            0
13158        );
13159        let mut replies = Vec::new();
13160        for _ in 0..requests {
13161            let mut restart = Box::pin(module.restart(Some(0)));
13162            // Poll the public API through queue admission, leaving only its ACK
13163            // pending. The monitor cannot consume the command until released.
13164            std::future::poll_fn(|cx| {
13165                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13166                std::task::Poll::Ready(())
13167            })
13168            .await;
13169            replies.push(restart);
13170        }
13171        release_tx.send(()).unwrap();
13172        startup.await.unwrap().unwrap();
13173        for reply in replies {
13174            reply.await.unwrap();
13175        }
13176        // Restart schedules its replacement after completing the handler. Wait
13177        // for that separate actor turn before placing the command barrier.
13178        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13179            loop {
13180                if lock_snapshot(&module.inner.snapshot)
13181                    .unwrap()
13182                    .spawn_generation
13183                    > 1
13184                {
13185                    break;
13186                }
13187                tokio::time::sleep(Duration::from_millis(1)).await;
13188            }
13189        })
13190        .await;
13191        // Commands are handled serially. The update-spec reply proves every
13192        // restart handler has finished before we inspect the snapshot; restart's
13193        // own ACK only acknowledges initiation, not replacement completion.
13194        module.update_spec_for_test(spec).await.unwrap();
13195        let (generation, pid) = {
13196            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13197            (state.spawn_generation, state.pid)
13198        };
13199        module.stop().await.unwrap();
13200        replacement.expect("restart must publish a replacement for the already-serving child");
13201        assert_ne!(
13202            pid,
13203            Some(old_pid),
13204            "a serving child predating the restart must be replaced"
13205        );
13206        assert_eq!(generation, 2, "queued restarts must share one replacement");
13207    }
13208
13209    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13210    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13211        restart_before_publication(1).await;
13212    }
13213
13214    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13215    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13216        restart_before_publication(2).await;
13217    }
13218
13219    #[tokio::test]
13220    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13221        let supervisor =
13222            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13223        let registered = supervisor
13224            .register_configured_with_health(
13225                ModuleSpec {
13226                    module_id: "queued-before-replacement".into(),
13227                    program: "/bin/sh".into(),
13228                    args: vec!["-c".into(), "exec sleep 600".into()],
13229                    env: vec![],
13230                    reserved: false,
13231                    reserved_prefixes: vec![],
13232                    protocol: ModuleProtocol::None,
13233                    overlap: Default::default(),
13234                },
13235                true,
13236                HealthConfig::default(),
13237                None,
13238                RestartPolicy::default(),
13239            )
13240            .unwrap();
13241        let RegisteredConfiguredModule {
13242            mut spec,
13243            mut runtime,
13244            module,
13245            mut commands,
13246        } = registered;
13247        let snapshot = &module.inner.snapshot;
13248        let mut restart = Box::pin(module.restart(Some(0)));
13249        std::future::poll_fn(|cx| {
13250            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13251            std::task::Poll::Ready(())
13252        })
13253        .await;
13254        // Drive the actor explicitly so a replacement is definitely spawned
13255        // after receipt, but before this queued command is dequeued.
13256        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13257        let replacement_pid = child.as_ref().unwrap().id();
13258        let command = commands.recv().await.unwrap();
13259        assert!(
13260            handle_supervisor_command(
13261                command,
13262                &mut spec,
13263                &mut runtime,
13264                &supervisor.registry,
13265                &supervisor.process_liveness,
13266                snapshot,
13267                &mut child,
13268                &mut commands,
13269                &mut VecDeque::new(),
13270            )
13271            .await
13272        );
13273        restart.await.unwrap();
13274        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13275        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13276        if let Some(mut child) = child {
13277            child.start_kill().unwrap();
13278            child.wait().await.unwrap();
13279        }
13280        assert_eq!(
13281            kept_pid, replacement_pid,
13282            "a newer replacement must survive"
13283        );
13284        assert!(
13285            !pending,
13286            "a satisfied restart must not schedule another spawn"
13287        );
13288    }
13289}
13290
13291#[cfg(all(test, unix))]
13292mod health_event_tests {
13293    use super::*;
13294    use cortexkit_test_support::ScratchDir;
13295    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13296
13297    struct Actor {
13298        module: SupervisedModule,
13299        registry: Arc<Registry>,
13300        forwarding: Arc<ForwardingTable>,
13301        spec: ModuleSpec,
13302        health: HealthConfig,
13303        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13304        _home: ScratchDir,
13305    }
13306
13307    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13308        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13309        // for an observed condition, rather than letting paused time auto-advance
13310        // deadlines or assuming a fixed number of yields finishes the work.
13311        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13312        let tick = Instant::now();
13313        loop {
13314            if let Some(value) = observe() {
13315                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13316                return value;
13317            }
13318            assert!(
13319                std::time::Instant::now() < deadline,
13320                "timed out waiting for {reason}"
13321            );
13322            tokio::task::yield_now().await;
13323        }
13324    }
13325
13326    impl Actor {
13327        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13328            let home = ScratchDir::new("health-events");
13329            let registry = Arc::new(Registry::default());
13330            let forwarding = Arc::new(ForwardingTable::default());
13331            let health = HealthConfig {
13332                cadence,
13333                deadline: Duration::from_secs(3600),
13334                ..HealthConfig::default()
13335            };
13336            let supervisor =
13337                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13338                    .with_forwarding(forwarding.clone())
13339                    .with_health_config(health.clone());
13340            let spec = ModuleSpec {
13341                module_id: "event-health-child".into(),
13342                program: PathBuf::from("/bin/sh"),
13343                // The process stays alive but has no real bus peer. Registrations
13344                // below exercise the same registry writes as accepted HELLOs.
13345                args: vec!["-c".into(), "exec sleep 600".into()],
13346                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13347                    .into_iter()
13348                    .map(|key| (key.into(), home.path().display().to_string()))
13349                    .collect(),
13350                reserved: false,
13351                reserved_prefixes: vec![],
13352                protocol,
13353                overlap: Default::default(),
13354            };
13355            let module = supervisor.spawn(spec.clone()).unwrap();
13356            let (_, rx) = mpsc::channel(8);
13357            let actor = Self {
13358                module,
13359                registry,
13360                forwarding,
13361                spec,
13362                health,
13363                rx,
13364                _home: home,
13365            };
13366            actor
13367                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13368                .await;
13369            actor
13370        }
13371
13372        fn turns(&self) -> u64 {
13373            lock_snapshot(&self.module.inner.snapshot)
13374                .unwrap()
13375                .actor_turns
13376        }
13377
13378        async fn wait_for_select(
13379            &self,
13380            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13381        ) -> ActorSelectCheckpoint {
13382            wait_for("actor select checkpoint", || {
13383                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13384                state.actor_select.filter(|checkpoint| {
13385                    checkpoint.generation == state.spawn_generation
13386                        && state.state == ModuleState::Running
13387                        && state.process_alive
13388                        && state.reported_pid().is_some()
13389                        && expected(checkpoint)
13390                })
13391            })
13392            .await
13393        }
13394
13395        fn hello(&mut self, connection: u64, health: bool) {
13396            let manifest =
13397                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13398                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13399                    .build();
13400            let connection = ConnectionId::new(connection);
13401            let (tx, rx) = mpsc::channel(8);
13402            self.rx = rx;
13403            self.forwarding
13404                .register_module_connection(
13405                    connection,
13406                    self.spec.module_id.clone(),
13407                    subc_protocol::PROTOCOL_VERSION,
13408                    Concurrency::ModuleManaged,
13409                    FrameSink::new(tx),
13410                )
13411                .unwrap();
13412            self.registry
13413                .register_with_control_ops(
13414                    manifest,
13415                    subc_protocol::PROTOCOL_VERSION,
13416                    connection,
13417                    if health {
13418                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13419                    } else {
13420                        vec![]
13421                    },
13422                )
13423                .unwrap();
13424        }
13425
13426        async fn probe(&mut self) -> crate::router::OutboundFrame {
13427            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13428                Ok(frame) => Some(frame),
13429                Err(mpsc::error::TryRecvError::Empty) => None,
13430                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13431            })
13432            .await;
13433            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13434            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13435            frame
13436        }
13437
13438        fn no_probe(&mut self) {
13439            assert!(matches!(
13440                self.rx.try_recv(),
13441                Err(mpsc::error::TryRecvError::Empty)
13442            ));
13443        }
13444
13445        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13446            self.forwarding
13447                .complete_module_control_rpc(
13448                    ConnectionId::new(connection),
13449                    frame.header.corr,
13450                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13451                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13452                        status: HealthStatus::Ok,
13453                        detail: None,
13454                        metrics: None,
13455                    }),
13456                )
13457                .unwrap();
13458        }
13459    }
13460
13461    #[tokio::test(start_paused = true)]
13462    async fn no_health_children_park_for_sixty_seconds() {
13463        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13464        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13465        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13466        unadvertised.hello(1, false);
13467        // Include any one-off registration wake in the budget; receiving a
13468        // no-health HELLO must not turn a parked child into a polling child.
13469        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13470        // a single 60 s jump would only observe one expired timer.
13471        for _ in 0..6000 {
13472            tokio::time::advance(Duration::from_millis(10)).await;
13473            tokio::task::yield_now().await;
13474        }
13475        for actor in [&none, &unregistered, &unadvertised] {
13476            assert!(
13477                actor.turns() <= 5,
13478                "no-health actor woke {} times",
13479                actor.turns()
13480            );
13481            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13482        }
13483    }
13484
13485    #[tokio::test(start_paused = true)]
13486    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13487        // With a zero cadence the first probe is due at once, so this measures
13488        // only how fast the registration notification starts health probing,
13489        // not the normal 30-33 s wait before a first probe (checked below).
13490        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13491        let tick = Instant::now();
13492        actor.hello(1, true);
13493        actor.probe().await;
13494        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13495    }
13496
13497    #[tokio::test(start_paused = true)]
13498    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13499        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13500        let hello_tick = Instant::now();
13501        actor.hello(1, true);
13502        let armed = actor
13503            .wait_for_select(|checkpoint| {
13504                checkpoint.registered_connection == Some(ConnectionId::new(1))
13505                    && checkpoint.next_probe_at.is_some()
13506            })
13507            .await;
13508        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13509        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13510        let first_deadline = hello_tick + delay;
13511        assert_eq!(armed.next_probe_at, Some(first_deadline));
13512        tokio::time::advance(Duration::from_secs(29)).await;
13513        let turns = actor.turns();
13514        actor
13515            .registry
13516            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13517            .unwrap();
13518        let updated = actor
13519            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13520            .await;
13521        assert_eq!(
13522            updated.next_probe_at,
13523            Some(first_deadline),
13524            "catalog update must not reset cadence"
13525        );
13526        actor.no_probe();
13527        tokio::time::advance(first_deadline - Instant::now()).await;
13528        let frame = actor.probe().await;
13529        actor.answer(1, frame);
13530        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13531        let next_deadline = Instant::now() + next;
13532        let rearmed = actor
13533            .wait_for_select(|checkpoint| {
13534                checkpoint
13535                    .next_probe_at
13536                    .is_some_and(|deadline| deadline > first_deadline)
13537            })
13538            .await;
13539        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13540        tokio::time::advance(next - Duration::from_nanos(1)).await;
13541        actor.no_probe();
13542        tokio::time::advance(Duration::from_nanos(1)).await;
13543        actor.probe().await;
13544    }
13545
13546    #[tokio::test(start_paused = true)]
13547    async fn disconnect_disarms_and_reconnect_rearms_health() {
13548        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13549        actor.hello(1, true);
13550        actor
13551            .wait_for_select(|checkpoint| {
13552                checkpoint.registered_connection == Some(ConnectionId::new(1))
13553                    && checkpoint.next_probe_at.is_some()
13554            })
13555            .await;
13556        let turns = actor.turns();
13557        let tick = Instant::now();
13558        actor
13559            .registry
13560            .deregister_connection(ConnectionId::new(1))
13561            .unwrap();
13562        let disarmed = actor
13563            .wait_for_select(|checkpoint| {
13564                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13565            })
13566            .await;
13567        assert_eq!(disarmed.next_probe_at, None);
13568        assert_eq!(disarmed.wake_after, None);
13569        assert_eq!(
13570            Instant::now(),
13571            tick,
13572            "disconnect must disarm in the notification tick"
13573        );
13574        tokio::time::advance(Duration::from_secs(60)).await;
13575        actor.no_probe();
13576        let reconnect_tick = Instant::now();
13577        actor.hello(2, true);
13578        let rearmed = actor
13579            .wait_for_select(|checkpoint| {
13580                checkpoint.registered_connection == Some(ConnectionId::new(2))
13581                    && checkpoint.next_probe_at.is_some()
13582            })
13583            .await;
13584        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13585        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13586        tokio::time::advance(delay).await;
13587        actor.probe().await;
13588    }
13589
13590    #[tokio::test(start_paused = true)]
13591    async fn reregistration_without_health_disarms_the_old_schedule() {
13592        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13593        actor.hello(1, true);
13594        actor
13595            .wait_for_select(|checkpoint| {
13596                checkpoint.registered_connection == Some(ConnectionId::new(1))
13597                    && checkpoint.next_probe_at.is_some()
13598            })
13599            .await;
13600        actor
13601            .registry
13602            .deregister_connection(ConnectionId::new(1))
13603            .unwrap();
13604        actor.hello(2, false);
13605        let disarmed = actor
13606            .wait_for_select(|checkpoint| {
13607                checkpoint.registered_connection == Some(ConnectionId::new(2))
13608            })
13609            .await;
13610        assert_eq!(disarmed.next_probe_at, None);
13611        assert_eq!(disarmed.wake_after, None);
13612        tokio::time::advance(Duration::from_secs(60)).await;
13613        actor.no_probe();
13614    }
13615
13616    #[tokio::test(start_paused = true)]
13617    async fn restart_waits_for_its_own_hello_before_rearming_health() {
13618        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13619        actor.hello(1, true);
13620        let armed = actor
13621            .wait_for_select(|checkpoint| {
13622                checkpoint.registered_connection == Some(ConnectionId::new(1))
13623                    && checkpoint.next_probe_at.is_some()
13624            })
13625            .await;
13626        // Simulate the old process's connection closing before the restart
13627        // tears the process down. The replacement process has started but has
13628        // not sent its HELLO, so it is not registered.
13629        actor
13630            .registry
13631            .deregister_connection(ConnectionId::new(1))
13632            .unwrap();
13633        actor.module.restart(Some(0)).await.unwrap();
13634        let parked = actor
13635            .wait_for_select(|checkpoint| {
13636                checkpoint.generation > armed.generation
13637                    && checkpoint.registered_connection.is_none()
13638                    && checkpoint.next_probe_at.is_none()
13639            })
13640            .await;
13641        assert_eq!(parked.wake_after, None);
13642        actor.rx.close();
13643        // Drain the stop notice the restart queued for the old process's connection,
13644        // so it isn't mistaken for traffic to the replacement.
13645        while actor.rx.try_recv().is_ok() {}
13646        let turns = actor.turns();
13647        tokio::time::advance(Duration::from_secs(60)).await;
13648        let after = actor.turns();
13649        eprintln!(
13650            "restart park turns: before={turns} after={after} delta={}",
13651            after - turns
13652        );
13653        assert_eq!(after, turns, "replacement must park before HELLO");
13654        let hello_tick = Instant::now();
13655        actor.hello(2, true);
13656        let rearmed = actor
13657            .wait_for_select(|checkpoint| {
13658                checkpoint.registered_connection == Some(ConnectionId::new(2))
13659                    && checkpoint.next_probe_at.is_some()
13660            })
13661            .await;
13662        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13663        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13664        tokio::time::advance(delay).await;
13665        actor.probe().await;
13666    }
13667
13668    #[tokio::test(start_paused = true)]
13669    async fn rescan_adds_and_removes_http_health_without_polling() {
13670        use tokio::{
13671            io::{AsyncReadExt, AsyncWriteExt},
13672            net::TcpListener,
13673        };
13674        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13675        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13676        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13677        // One millisecond separates successive probes without a polling delay.
13678        actor.health.cadence = Duration::from_millis(1);
13679        let (started_tx, mut started_rx) = mpsc::channel(8);
13680        let peer = tokio::spawn(async move {
13681            loop {
13682                let (mut socket, _) = listener.accept().await.unwrap();
13683                let mut request = [0; 1024];
13684                assert!(socket.read(&mut request).await.unwrap() > 0);
13685                started_tx.send(()).await.unwrap();
13686                socket
13687                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13688                    .await
13689                    .unwrap();
13690            }
13691        });
13692        let tick = Instant::now();
13693        let turns = actor.turns();
13694        actor
13695            .module
13696            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13697            .await
13698            .unwrap();
13699        let armed = actor
13700            .wait_for_select(|checkpoint| {
13701                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13702            })
13703            .await;
13704        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13705        assert_eq!(Instant::now(), tick);
13706        tokio::time::advance(Duration::from_millis(1)).await;
13707        let mut started = false;
13708        wait_for("successful HTTP probe", || {
13709            started |= started_rx.try_recv().is_ok();
13710            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13711                .then_some(())
13712        })
13713        .await;
13714        assert!(started, "added HTTP check must start at its first deadline");
13715        assert_eq!(
13716            actor.module.status().unwrap().health.status,
13717            SupervisorHealthStatus::Ok
13718        );
13719        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13720        actor.health.http = None;
13721        let turns = actor.turns();
13722        actor
13723            .module
13724            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13725            .await
13726            .unwrap();
13727        let parked = actor
13728            .wait_for_select(|checkpoint| {
13729                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13730            })
13731            .await;
13732        assert_eq!(parked.wake_after, None);
13733        let turns = actor.turns();
13734        tokio::time::advance(Duration::from_secs(60)).await;
13735        assert_eq!(
13736            actor.turns(),
13737            turns,
13738            "removed HTTP check must park the actor"
13739        );
13740        assert!(
13741            started_rx.try_recv().is_err(),
13742            "removed HTTP check must stay disarmed"
13743        );
13744        assert_eq!(
13745            actor.module.status().unwrap().health,
13746            ModuleHealthStatus::default()
13747        );
13748        peer.abort();
13749    }
13750}
13751
13752#[cfg(test)]
13753mod drain_busy_reprobe_tests {
13754    //! The drain's busy-gauge wait, driven against an in-process module
13755    //! connection whose health answers the test scripts. Time is paused, so
13756    //! every elapsed duration below is exact virtual time.
13757    use super::*;
13758    use std::sync::Mutex as StdMutex;
13759    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13760
13761    const MODULE_ID: &str = "drain-reprobe-module";
13762    const CONNECTION: u64 = 7;
13763    const GAUGE: &str = "runs_in_flight";
13764    /// The production default for both the health cadence and the drain
13765    /// ceiling, which is the pairing that used to cost a full ceiling.
13766    const CADENCE: Duration = Duration::from_secs(30);
13767    const CEILING: Duration = Duration::from_secs(30);
13768
13769    /// How the scripted module answers each `health.check`.
13770    #[derive(Clone, Copy)]
13771    enum Answer {
13772        /// Report the gauge busy until this instant, quiet from then on, after
13773        /// taking `latency` to answer.
13774        BusyUntil { until: Instant, latency: Duration },
13775        /// Never answer.
13776        Silent,
13777    }
13778
13779    struct Drain {
13780        forwarding: Arc<ForwardingTable>,
13781        runtime: SupervisorRuntimeConfig,
13782        endpoint: crate::ModuleEndpointId,
13783        /// When each probe reached the module.
13784        probes: Arc<StdMutex<Vec<Instant>>>,
13785        responder: tokio::task::JoinHandle<()>,
13786    }
13787
13788    impl Drain {
13789        fn start(answer: Answer) -> Self {
13790            let registry = Arc::new(Registry::default());
13791            let forwarding = Arc::new(ForwardingTable::default());
13792            let health = HealthConfig {
13793                cadence: CADENCE,
13794                ..HealthConfig::default()
13795            };
13796            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13797                .with_forwarding(forwarding.clone())
13798                .with_health_config(health)
13799                .runtime_config();
13800            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13801            let endpoint = forwarding
13802                .register_module_connection(
13803                    ConnectionId::new(CONNECTION),
13804                    MODULE_ID.to_string(),
13805                    subc_protocol::PROTOCOL_VERSION,
13806                    Concurrency::ModuleManaged,
13807                    FrameSink::new(tx),
13808                )
13809                .unwrap();
13810            let probes = Arc::new(StdMutex::new(Vec::new()));
13811            let responder = tokio::spawn({
13812                let forwarding = forwarding.clone();
13813                let probes = probes.clone();
13814                async move {
13815                    while let Some(frame) = rx.recv().await {
13816                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
13817                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13818                        probes.lock().unwrap().push(Instant::now());
13819                        let Answer::BusyUntil { until, latency } = answer else {
13820                            continue;
13821                        };
13822                        sleep(latency).await;
13823                        let busy = u64::from(Instant::now() < until);
13824                        // A probe abandoned at its deadline has no RPC left to
13825                        // complete, so a late answer may be refused.
13826                        let _ = forwarding.complete_module_control_rpc(
13827                            ConnectionId::new(CONNECTION),
13828                            frame.header.corr,
13829                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13830                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13831                                status: HealthStatus::Ok,
13832                                detail: None,
13833                                metrics: Some(serde_json::json!({ GAUGE: busy })),
13834                            }),
13835                        );
13836                    }
13837                }
13838            });
13839            Self {
13840                forwarding,
13841                runtime,
13842                endpoint,
13843                probes,
13844                responder,
13845            }
13846        }
13847
13848        /// Run one by-id drain wait with `ceiling`, returning its verdict and
13849        /// how long it took.
13850        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
13851            let started = Instant::now();
13852            let drained = wait_for_forwarding_quiescence(
13853                &self.forwarding,
13854                MODULE_ID,
13855                &self.runtime,
13856                self.endpoint,
13857                started + ceiling,
13858                &[GAUGE.to_string()],
13859                DrainScope::Active,
13860            )
13861            .await
13862            .unwrap();
13863            (drained, started.elapsed())
13864        }
13865
13866        fn probes(&self) -> Vec<Instant> {
13867            self.probes.lock().unwrap().clone()
13868        }
13869    }
13870
13871    impl Drop for Drain {
13872        fn drop(&mut self) {
13873            self.responder.abort();
13874        }
13875    }
13876
13877    #[tokio::test(start_paused = true)]
13878    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
13879        let drain = Drain::start(Answer::BusyUntil {
13880            until: Instant::now() + Duration::from_secs(1),
13881            latency: Duration::ZERO,
13882        });
13883        let (drained, elapsed) = drain.run(CEILING).await;
13884        assert!(
13885            drained,
13886            "gauges that read zero during the drain must let it finish drained, \
13887             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
13888        );
13889        assert!(
13890            elapsed < Duration::from_secs(3),
13891            "drain took {elapsed:?} after the gauges went quiet at 1s"
13892        );
13893        let probes = drain.probes();
13894        assert!(
13895            probes.len() >= 2,
13896            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
13897        );
13898    }
13899
13900    #[tokio::test(start_paused = true)]
13901    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
13902        let started = Instant::now();
13903        let drain = Drain::start(Answer::BusyUntil {
13904            until: started + Duration::from_secs(3600),
13905            latency: Duration::ZERO,
13906        });
13907        let (drained, elapsed) = drain.run(CEILING).await;
13908        assert!(!drained, "a module busy throughout must not drain");
13909        assert!(
13910            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
13911            "a busy module is held until the ceiling and no longer: {elapsed:?}"
13912        );
13913        // Re-probing is paced by the interval, not a busy loop: at most one
13914        // probe per interval across the ceiling, plus the first at time zero.
13915        let probes = drain.probes();
13916        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
13917        assert!(
13918            probes.len() as u128 <= max_probes,
13919            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
13920            probes.len()
13921        );
13922        for pair in probes.windows(2) {
13923            assert!(
13924                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
13925                "consecutive probes {:?} apart",
13926                pair[1] - pair[0]
13927            );
13928        }
13929    }
13930
13931    #[tokio::test(start_paused = true)]
13932    async fn a_slow_probe_is_never_overlapped_by_the_next() {
13933        // Each answer takes longer than the re-probe interval; the next probe
13934        // must wait for the previous answer and then the interval.
13935        let latency = Duration::from_secs(1);
13936        let drain = Drain::start(Answer::BusyUntil {
13937            until: Instant::now() + Duration::from_secs(3600),
13938            latency,
13939        });
13940        let (drained, _) = drain.run(Duration::from_secs(5)).await;
13941        assert!(!drained);
13942        let probes = drain.probes();
13943        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
13944        for pair in probes.windows(2) {
13945            assert!(
13946                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
13947                "a probe started {:?} after the previous one, before it could finish",
13948                pair[1] - pair[0]
13949            );
13950        }
13951    }
13952
13953    #[tokio::test(start_paused = true)]
13954    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
13955        let drain = Drain::start(Answer::Silent);
13956        let ceiling = Duration::from_secs(2);
13957        let (drained, elapsed) = drain.run(ceiling).await;
13958        assert!(!drained, "an unanswered probe counts as busy");
13959        assert!(
13960            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
13961            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
13962        );
13963        assert_eq!(
13964            drain.probes().len(),
13965            1,
13966            "the unanswered probe holds the only slot until the ceiling"
13967        );
13968    }
13969}
13970
13971#[cfg(test)]
13972mod health_evidence_tests {
13973    use super::{HealthProbeError, HealthProbeEvidence};
13974    use std::collections::HashSet;
13975
13976    /// The evidential asymmetry, asserted rather than described.
13977    ///
13978    /// Exactly ONE observation is proof a module cannot serve, and the one that
13979    /// fires under CPU starvation is not it. Before the split, all fifteen
13980    /// construction sites collapsed into a single String, so a timeout carried the
13981    /// same weight as a dead lane -- which is how a healthy module was restarted
13982    /// three times in one day.
13983    #[test]
13984    fn only_a_dead_lane_is_proof_of_death() {
13985        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13986        // Three non-proof classes, each for a different reason: silence is
13987        // consistent with health, a bad answer proves the module ALIVE, and a
13988        // daemon-side fault never reached the module at all.
13989        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13990        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13991        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13992    }
13993
13994    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
13995    ///
13996    /// A shared label renders two different observations identically in the line an
13997    /// operator reads after an unexplained restart -- the exact confusion this
13998    /// change removes.
13999    #[test]
14000    fn every_evidence_class_has_a_distinct_label() {
14001        let labels = [
14002            HealthProbeError::lane_dead("").label(),
14003            HealthProbeError::no_answer("").label(),
14004            HealthProbeError::bad_answer("").label(),
14005            HealthProbeError::misconfigured("").label(),
14006        ];
14007        let unique: HashSet<_> = labels.iter().collect();
14008        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14009    }
14010
14011    /// The class is additional information, not a replacement.
14012    ///
14013    /// An operator needs both "this was silence" and the specific text saying how
14014    /// long we waited; a classification that swallowed the message would trade one
14015    /// missing distinction for another.
14016    #[test]
14017    fn classification_preserves_the_original_message() {
14018        let err = HealthProbeError::no_answer("module did not answer within 5s");
14019        assert_eq!(err.to_string(), "module did not answer within 5s");
14020        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14021    }
14022}
14023
14024#[cfg(test)]
14025mod health_tombstone_tests {
14026    use std::{path::PathBuf, sync::Arc, time::Duration};
14027
14028    use subc_protocol::{
14029        manifest::Concurrency,
14030        session::{HealthStatus, ModuleControlResponse},
14031    };
14032    use tokio::sync::mpsc;
14033
14034    use super::{
14035        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14036        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14037    };
14038    use crate::{
14039        control::ControlHandler,
14040        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14041        registry::{ConnectionId, Registry},
14042        router::FrameSink,
14043    };
14044
14045    struct ProbeHarness {
14046        spec: ModuleSpec,
14047        runtime: SupervisorRuntimeConfig,
14048        forwarding: Arc<ForwardingTable>,
14049        module_connection: ConnectionId,
14050        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14051        handler: ControlHandler,
14052        module: super::SupervisedModule,
14053    }
14054
14055    fn probe_harness() -> ProbeHarness {
14056        let registry = Arc::new(Registry::default());
14057        let forwarding = Arc::new(ForwardingTable::default());
14058        let supervisor_handle = super::SupervisorHandle::new();
14059        let health = HealthConfig {
14060            http: None,
14061            cadence: Duration::from_secs(30),
14062            deadline: Duration::from_secs(5),
14063            failure_threshold: 3,
14064            on_degraded: HealthAction::Report,
14065            on_failing: HealthAction::Report,
14066            critical: false,
14067        };
14068        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14069            .with_forwarding(Arc::clone(&forwarding))
14070            .with_handle(supervisor_handle.clone())
14071            .with_health_config(health);
14072        let spec = ModuleSpec {
14073            module_id: "late-health-module".to_string(),
14074            program: PathBuf::from("disabled-module"),
14075            args: Vec::new(),
14076            env: Vec::new(),
14077            reserved: false,
14078            reserved_prefixes: Vec::new(),
14079            protocol: ModuleProtocol::Subc,
14080            overlap: Default::default(),
14081        };
14082        let module = supervisor
14083            .supervise_configured(spec.clone(), false)
14084            .unwrap();
14085        let runtime = supervisor.runtime_config();
14086        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14087            .with_supervisor(supervisor_handle);
14088        let module_connection = ConnectionId::new(700);
14089        let (module_tx, module_rx) = mpsc::channel(8);
14090        forwarding
14091            .register_module_connection(
14092                module_connection,
14093                spec.module_id.clone(),
14094                subc_protocol::PROTOCOL_VERSION,
14095                Concurrency::ModuleManaged,
14096                FrameSink::new(module_tx),
14097            )
14098            .unwrap();
14099
14100        ProbeHarness {
14101            spec,
14102            runtime,
14103            forwarding,
14104            module_connection,
14105            module_rx,
14106            handler,
14107            module,
14108        }
14109    }
14110
14111    async fn finish_after(
14112        harness: &mut ProbeHarness,
14113        stall: Duration,
14114    ) -> ModuleControlRpcCompletion {
14115        assert!(stall > harness.runtime.health.deadline);
14116        let deadline = harness.runtime.health.deadline;
14117        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14118        let answer = async {
14119            let frame = harness.module_rx.recv().await.expect("health.check frame");
14120            tokio::time::advance(deadline).await;
14121            tokio::task::yield_now().await;
14122            tokio::time::advance(stall - deadline).await;
14123            harness
14124                .forwarding
14125                .complete_module_control_rpc(
14126                    harness.module_connection,
14127                    frame.header.corr,
14128                    Some("health.check"),
14129                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14130                        status: HealthStatus::Ok,
14131                        detail: None,
14132                        metrics: None,
14133                    }),
14134                )
14135                .unwrap()
14136        };
14137        let (probe_result, completion) = tokio::join!(probe, answer);
14138        let err = probe_result.expect_err("probe must miss its deadline");
14139        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14140        completion
14141    }
14142
14143    async fn time_out_without_answer(harness: &mut ProbeHarness) {
14144        let deadline = harness.runtime.health.deadline;
14145        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14146        let exhaust_deadline = async {
14147            let _frame = harness.module_rx.recv().await.expect("health.check frame");
14148            tokio::time::advance(deadline).await;
14149            tokio::task::yield_now().await;
14150        };
14151        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14152        let err = probe_result.expect_err("probe must miss its deadline");
14153        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14154    }
14155
14156    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14157        let registry = Arc::clone(&harness.module.inner.registry);
14158        let snapshot = Arc::clone(&harness.module.inner.snapshot);
14159        let process_liveness = super::SupervisorProcessLiveness::default();
14160        let mut child = None;
14161        let cycle = super::run_health_probe_cycle(
14162            &harness.spec,
14163            &harness.runtime,
14164            &registry,
14165            &process_liveness,
14166            &snapshot,
14167            &mut child,
14168        );
14169        let peer = async {
14170            let frame = harness.module_rx.recv().await.expect("health.check frame");
14171            if answer {
14172                harness
14173                    .forwarding
14174                    .complete_module_control_rpc(
14175                        harness.module_connection,
14176                        frame.header.corr,
14177                        Some("health.check"),
14178                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14179                            status: HealthStatus::Ok,
14180                            detail: None,
14181                            metrics: Some(serde_json::json!({"ready": true})),
14182                        }),
14183                    )
14184                    .unwrap();
14185            } else {
14186                tokio::time::advance(harness.runtime.health.deadline).await;
14187                tokio::task::yield_now().await;
14188            }
14189        };
14190        tokio::join!(cycle, peer);
14191    }
14192
14193    #[tokio::test(start_paused = true)]
14194    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14195        let mut harness = probe_harness();
14196        // Drive the probe cycle directly with an in-memory wire peer. Stop the
14197        // disabled module's monitor so only this test owns lifecycle transitions;
14198        // no OS process is launched, and a restart is observed at scheduling.
14199        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14200        monitor.abort();
14201        let _ = monitor.await;
14202        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14203            state.enabled = true;
14204            state.state = super::ModuleState::Running;
14205            state.process_alive = true;
14206        })
14207        .unwrap();
14208
14209        run_probe_cycle(&mut harness, true).await;
14210        assert_eq!(
14211            harness.module.status().unwrap().health.status,
14212            super::SupervisorHealthStatus::Ok
14213        );
14214
14215        for failures in 1..harness.runtime.health.failure_threshold {
14216            run_probe_cycle(&mut harness, false).await;
14217            let status = harness.module.status().unwrap();
14218            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14219            assert_eq!(status.health.consecutive_failures, failures);
14220            assert!(status.health.last_probe_ms.is_some());
14221            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14222            assert!(status.health.metrics.is_none());
14223            assert_eq!(status.state, super::ModuleState::Running);
14224            assert!(status.process_alive);
14225            assert_eq!(status.restart_count, 0);
14226            assert_eq!(status.lifetime_restarts, 0);
14227            assert!(status.health.last_action.is_none());
14228            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14229        }
14230
14231        run_probe_cycle(&mut harness, true).await;
14232        let recovered = harness.module.status().unwrap();
14233        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14234        assert_eq!(recovered.health.consecutive_failures, 0);
14235        assert!(recovered.health.detail.is_none());
14236        assert_eq!(
14237            recovered.health.metrics,
14238            Some(serde_json::json!({"ready": true}))
14239        );
14240        assert_eq!(recovered.lifetime_restarts, 0);
14241
14242        for failures in 1..=harness.runtime.health.failure_threshold {
14243            run_probe_cycle(&mut harness, false).await;
14244            let status = harness.module.status().unwrap();
14245            assert_eq!(status.health.consecutive_failures, failures);
14246            if failures < harness.runtime.health.failure_threshold {
14247                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14248                assert_eq!(status.state, super::ModuleState::Running);
14249                assert_eq!(status.lifetime_restarts, 0);
14250                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14251            } else {
14252                assert_eq!(
14253                    status.health.status,
14254                    super::SupervisorHealthStatus::Unresponsive
14255                );
14256                assert_eq!(status.state, super::ModuleState::Restarting);
14257                assert_eq!(status.restart_count, 1);
14258                assert_eq!(status.lifetime_restarts, 1);
14259                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14260                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14261            }
14262        }
14263    }
14264
14265    #[tokio::test(start_paused = true)]
14266    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14267        let mut harness = probe_harness();
14268
14269        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14270        let first_latency = match &first {
14271            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14272            other => panic!("late answer was not retained: {other:?}"),
14273        };
14274        assert!(harness.handler.observe_module_control_completion(first));
14275
14276        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14277        let second_latency = match &second {
14278            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14279            other => panic!("late answer was not retained: {other:?}"),
14280        };
14281        assert!(harness.handler.observe_module_control_completion(second));
14282
14283        assert_eq!(first_latency, Duration::from_secs(8));
14284        assert_eq!(
14285            second_latency - first_latency,
14286            Duration::from_secs(3),
14287            "latency must grow linearly with the additional stall"
14288        );
14289        let health = harness.module.status().unwrap().health;
14290        assert_eq!(health.late_answer_count, 2);
14291        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14292    }
14293
14294    /// A module that answers every probe late must never march to the kill
14295    /// threshold: the late answer proves it is alive, so it must clear the miss
14296    /// streak the timeout recorded. Without the reset, a CPU-starved module
14297    /// that serves every probe seconds past the deadline accumulates
14298    /// `consecutive_failures` to the threshold and is killed — the exact
14299    /// sequence from the 2026-08-14 aft disable, where the daemon logged
14300    /// "proves the module is alive" five times while counting five misses.
14301    #[tokio::test(start_paused = true)]
14302    async fn late_answer_clears_the_consecutive_failure_streak() {
14303        let mut harness = probe_harness();
14304
14305        // Timeout recorded first: the probe path saw no answer in time.
14306        time_out_without_answer(&mut harness).await;
14307        harness
14308            .module
14309            .record_health_probe_failure_for_test("[no-answer] test miss")
14310            .unwrap();
14311        assert_eq!(
14312            harness.module.status().unwrap().health.consecutive_failures,
14313            1,
14314            "precondition: the miss must be on the streak before the late answer"
14315        );
14316
14317        // The stalled reply then lands: proof of life.
14318        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14319        assert!(matches!(
14320            late,
14321            ModuleControlRpcCompletion::LateHealthAnswer { .. }
14322        ));
14323        assert!(harness.handler.observe_module_control_completion(late));
14324
14325        let health = harness.module.status().unwrap().health;
14326        assert_eq!(
14327            health.consecutive_failures, 0,
14328            "a late answer is an answer: the streak must reset"
14329        );
14330        assert_eq!(health.late_answer_count, 1);
14331    }
14332
14333    #[tokio::test(start_paused = true)]
14334    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14335        let mut harness = probe_harness();
14336
14337        for _ in 0..20 {
14338            time_out_without_answer(&mut harness).await;
14339            assert_eq!(
14340                harness.forwarding.health_probe_tombstone_count().unwrap(),
14341                1
14342            );
14343        }
14344    }
14345}
14346
14347#[cfg(test)]
14348mod child_env_tests {
14349    #[cfg(windows)]
14350    use super::{
14351        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14352    };
14353    #[cfg(windows)]
14354    use std::sync::{Arc, OnceLock};
14355    #[cfg(windows)]
14356    #[test]
14357    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14358        let spec = spec(Vec::new());
14359        let mut command = Command::new("unused");
14360        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14361            .unwrap()
14362            .unwrap();
14363        let name = command
14364            .as_std()
14365            .get_envs()
14366            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14367            .and_then(|(_, value)| value)
14368            .unwrap();
14369        assert_eq!(name, OsStr::new(handoff.name()));
14370        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14371        assert!(command.as_std().get_envs().any(|(key, value)| key
14372            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14373            && value == Some(OsStr::new("env"))));
14374        assert!(command
14375            .as_std()
14376            .get_envs()
14377            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14378    }
14379
14380    #[cfg(windows)]
14381    #[test]
14382    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14383        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14384        snapshot.process_alive = true;
14385        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14386        assert_eq!(
14387            observed_launch_nonce_source(&snapshot).as_deref(),
14388            Some("env")
14389        );
14390        let consumed = Arc::new(OnceLock::new());
14391        snapshot.pipe_consumed = Some(consumed.clone());
14392        assert_eq!(
14393            observed_launch_nonce_source(&snapshot).as_deref(),
14394            Some("env")
14395        );
14396        consumed.set(()).unwrap();
14397        assert_eq!(
14398            observed_launch_nonce_source(&snapshot).as_deref(),
14399            Some("pipe")
14400        );
14401        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14402        assert_eq!(
14403            observed_launch_nonce_source(&snapshot).as_deref(),
14404            Some("env")
14405        );
14406        clear_current_process_facts(&mut snapshot);
14407        assert_eq!(observed_launch_nonce_source(&snapshot), None);
14408    }
14409    use super::{
14410        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14411        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14412        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14413    };
14414    use std::{ffi::OsStr, path::PathBuf};
14415    use tokio::process::Command;
14416
14417    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14418        ModuleSpec {
14419            module_id: "env-plan".to_string(),
14420            program: PathBuf::from("/nonexistent"),
14421            args: Vec::new(),
14422            env,
14423            reserved: false,
14424            reserved_prefixes: Vec::new(),
14425            protocol: ModuleProtocol::Subc,
14426            overlap: Default::default(),
14427        }
14428    }
14429
14430    #[test]
14431    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14432        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14433        let digest = LaunchSpecDigest::from_spec(&base);
14434        let mut reordered = base.clone();
14435        reordered.env.reverse();
14436        reordered.module_id = "different-id".into();
14437        reordered.reserved = true;
14438        reordered.overlap = Default::default();
14439        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14440        let mut delimiter_collision = base.clone();
14441        delimiter_collision.env[0] = ("A=B".into(), "C".into());
14442        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14443        let mut changed = base.clone();
14444        changed.protocol = ModuleProtocol::None;
14445        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14446        changed = base.clone();
14447        changed.args.push("--flag".into());
14448        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14449        changed = base.clone();
14450        changed.program = PathBuf::from("/other");
14451        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14452        changed = base.clone();
14453        changed.env[1].1 = "other".into();
14454        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14455        assert!(!format!("{digest:?}").contains("B=C"));
14456        assert!(!format!("{digest:?}").contains("[u8"));
14457    }
14458
14459    #[cfg(unix)]
14460    #[test]
14461    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14462        use std::os::unix::ffi::OsStringExt;
14463        let mut left = spec(Vec::new());
14464        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14465        let mut right = left.clone();
14466        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14467        assert_eq!(
14468            left.program.to_string_lossy(),
14469            right.program.to_string_lossy()
14470        );
14471        assert_ne!(
14472            LaunchSpecDigest::from_spec(&left),
14473            LaunchSpecDigest::from_spec(&right)
14474        );
14475    }
14476
14477    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
14478    /// one still gets its own.
14479    ///
14480    /// This is the narrow goal `env_clear()` was reached for, and the reason the
14481    /// fix is `env_remove` rather than deleting the line: an operator's ambient
14482    /// filter silently becoming an unconfigured module's log level is a real
14483    /// defect, just a much smaller one than clearing the environment.
14484    ///
14485    /// Asserted on the command plan rather than a spawned child because proving
14486    /// the ABSENCE of an inherited variable needs the parent's environment
14487    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
14488    /// removal as `(key, None)`, which is exactly the distinction wanted: not
14489    /// "absent because nobody set it" but "explicitly unset for the child".
14490    #[test]
14491    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14492        let mut command = Command::new("/nonexistent");
14493        apply_child_env(&mut command, &spec(Vec::new()));
14494        let removed = command
14495            .as_std()
14496            .get_envs()
14497            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14498        assert!(
14499            removed,
14500            "ambient CK_LOG must be explicitly removed for an unconfigured module"
14501        );
14502
14503        let mut configured = Command::new("/nonexistent");
14504        apply_child_env(
14505            &mut configured,
14506            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14507        );
14508        let effective = configured
14509            .as_std()
14510            .get_envs()
14511            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14512            .last()
14513            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14514        assert_eq!(
14515            effective,
14516            Some(Some("debug".to_string())),
14517            "a module's configured CK_LOG must survive the ambient removal"
14518        );
14519    }
14520
14521    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
14522    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
14523    /// the same reason as the CK_LOG test above.
14524    ///
14525    /// The argument is the load-bearing half: a stock binary exits on an
14526    /// unknown flag before it listens, so with `--subc` appended the mode
14527    /// could not supervise the one process it exists for. Found by the first
14528    /// conformance run (nats-server: `flag provided but not defined: -subc`).
14529    #[test]
14530    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14531        let connection_file = std::path::Path::new("/run/subc-connection.json");
14532        let handle = SupervisorHandle::new();
14533
14534        let mut none_spec = spec(Vec::new());
14535        none_spec.protocol = ModuleProtocol::None;
14536        let mut none = Command::new("/nonexistent");
14537        let none_handoff =
14538            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14539                .expect("protocol-none spawn args apply");
14540        assert!(
14541            none_handoff.is_none(),
14542            "protocol:none spawn must not receive a nonce descriptor"
14543        );
14544        #[cfg(windows)]
14545        assert!(!none.as_std().get_envs().any(|(key, value)| key
14546            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14547            && value.is_some()));
14548        assert!(
14549            !none.as_std().get_envs().any(|(key, value)| key
14550                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14551                && value.is_some()),
14552            "protocol:none spawn must not name a nonce descriptor"
14553        );
14554        let none_args: Vec<String> = none
14555            .as_std()
14556            .get_args()
14557            .map(|a| a.to_string_lossy().into_owned())
14558            .collect();
14559        assert!(
14560            !none_args.iter().any(|a| a == SUBC_ARG),
14561            "protocol:none argv must not carry --subc; got {none_args:?}"
14562        );
14563        let none_has_nonce = none
14564            .as_std()
14565            .get_envs()
14566            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14567        assert!(
14568            !none_has_nonce,
14569            "protocol:none spawn must not receive a launch nonce"
14570        );
14571        let none_has_module_id = none
14572            .as_std()
14573            .get_envs()
14574            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14575        assert!(
14576            none_has_module_id,
14577            "SUBC_MODULE_ID is inert and stays on every path"
14578        );
14579        assert!(
14580            handle.spawn_nonce(&none_spec.module_id).is_none(),
14581            "no nonce record for a process that will never present one"
14582        );
14583
14584        // Control: the subc-wire path is unchanged by the branch above.
14585        let wire_spec = spec(Vec::new());
14586        let mut wire = Command::new("/nonexistent");
14587        let wire_handoff =
14588            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14589                .expect("subc-wire spawn args apply");
14590        let wire_fd_env = wire
14591            .as_std()
14592            .get_envs()
14593            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14594            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14595        #[cfg(unix)]
14596        assert_eq!(
14597            wire_fd_env,
14598            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14599            "a subc-wire spawn names the pipe it will receive at descriptor 3"
14600        );
14601        #[cfg(windows)]
14602        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14603        #[cfg(not(any(unix, windows)))]
14604        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14605        let wire_args: Vec<String> = wire
14606            .as_std()
14607            .get_args()
14608            .map(|a| a.to_string_lossy().into_owned())
14609            .collect();
14610        assert_eq!(
14611            wire_args,
14612            vec![
14613                SUBC_ARG.to_string(),
14614                connection_file.to_string_lossy().into_owned()
14615            ],
14616            "a subc-wire spawn still carries --subc <path>"
14617        );
14618        assert_eq!(
14619            wire.as_std()
14620                .get_envs()
14621                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14622            !cfg!(unix),
14623            "only Windows supplies the environment nonce"
14624        );
14625        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14626    }
14627
14628    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
14629    /// spec tries to set it; only a swap candidate carries it.
14630    ///
14631    /// "Set it only on candidates" is not enough, because spawn applies the
14632    /// spec's env verbatim and the daemon's own environment is inherited: either
14633    /// could hand a plain restart the swap role, and a module reading it would
14634    /// warm on its long swap budget while callers wait. Asserted as an explicit
14635    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
14636    /// test above gives.
14637    #[test]
14638    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14639        let role = |command: &Command| {
14640            command
14641                .as_std()
14642                .get_envs()
14643                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14644                .last()
14645                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14646        };
14647        let forged = spec(vec![(
14648            SUBC_SPAWN_ROLE_ENV.to_string(),
14649            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14650        )]);
14651
14652        let mut plain = Command::new("/nonexistent");
14653        apply_child_env(&mut plain, &forged);
14654        apply_spawn_role(&mut plain, SpawnRole::Plain);
14655        assert_eq!(
14656            role(&plain),
14657            Some(None),
14658            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14659        );
14660
14661        let mut candidate = Command::new("/nonexistent");
14662        apply_child_env(&mut candidate, &spec(Vec::new()));
14663        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14664        assert_eq!(
14665            role(&candidate),
14666            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14667        );
14668    }
14669
14670    /// Daemon-private capture retention keys never reach the child.
14671    ///
14672    /// cortexkit-log exposes retention as a Rust struct with no environment
14673    /// names, so these entries are supervisor metadata. Passing them through
14674    /// would invent a public child-process contract by accident.
14675    #[test]
14676    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14677        let mut command = Command::new("/nonexistent");
14678        apply_child_env(
14679            &mut command,
14680            &spec(vec![
14681                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14682                ("KEPT".to_string(), "yes".to_string()),
14683            ]),
14684        );
14685        let keys: Vec<String> = command
14686            .as_std()
14687            .get_envs()
14688            .filter(|(_, value)| value.is_some())
14689            .map(|(key, _)| key.to_string_lossy().into_owned())
14690            .collect();
14691        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14692        assert!(
14693            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14694            "daemon-private capture key leaked to the child: {keys:?}"
14695        );
14696    }
14697}
14698
14699#[cfg(test)]
14700mod jitter_tests {
14701    use super::jittered_health_delay;
14702    use std::{collections::HashSet, time::Duration};
14703
14704    /// Module ids drawn from a real fleet, so the dispersal claim is about names
14705    /// that actually occur rather than invented ones.
14706    ///
14707    /// This is a SAMPLE, not a registry: the property under test is that distinct
14708    /// ids disperse, which holds for any set of distinct strings. Several entries
14709    /// are already historical (modules get renamed), and that costs nothing here --
14710    /// but it means a reader must not mistake this for the live module set, and a
14711    /// rename sweep will match it without there being anything to change.
14712    const FLEET: [&str; 14] = [
14713        "aft",
14714        "alfonso-core",
14715        "magic-context",
14716        "broca",
14717        "thalamus",
14718        "quota",
14719        "engram",
14720        "plexus",
14721        "cerebellum",
14722        "astrocyte",
14723        "synapse",
14724        "subc-mcp",
14725        "cortexkit-credentials",
14726        "subc-federation",
14727    ];
14728
14729    /// Probes must not converge after a fleet-wide restart.
14730    ///
14731    /// This is the property the jitter exists for: every module reconnects at
14732    /// once, and without dispersal all fourteen would then probe on the same
14733    /// tick forever. Nothing failed visibly when this went untested -- a
14734    /// convergent fleet still probes correctly, just in a burst, so the symptom
14735    /// is a periodic load spike that looks like whatever else is running.
14736    #[test]
14737    fn probe_delays_disperse_across_the_fleet() {
14738        let cadence = Duration::from_secs(30);
14739        let delays: HashSet<Duration> = FLEET
14740            .iter()
14741            .map(|id| jittered_health_delay(id, 0, cadence))
14742            .collect();
14743        assert_eq!(
14744            delays.len(),
14745            FLEET.len(),
14746            "every supervised module must land on its own probe offset"
14747        );
14748    }
14749
14750    /// The offset may only ever DELAY a probe, never bring it forward.
14751    ///
14752    /// A delay below the cadence would probe a module more often than
14753    /// configured, which is the opposite of what an operator asked for and
14754    /// would tighten the failure budget without anyone changing it.
14755    #[test]
14756    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14757        let cadence = Duration::from_secs(30);
14758        let span = cadence / 10;
14759        for id in FLEET {
14760            for probe_index in 0..8 {
14761                let delay = jittered_health_delay(id, probe_index, cadence);
14762                assert!(
14763                    delay >= cadence,
14764                    "{id}#{probe_index}: jitter must not shorten the cadence"
14765                );
14766                assert!(
14767                    delay < cadence + span,
14768                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14769                );
14770            }
14771        }
14772    }
14773
14774    /// A module keeps its offset across daemon restarts.
14775    ///
14776    /// The delay is derived rather than randomised precisely so a restart does
14777    /// not re-roll every module into a fresh chance of collision. A random
14778    /// source would satisfy the dispersal test above and quietly lose this.
14779    #[test]
14780    fn a_module_offset_is_stable_across_restarts() {
14781        let cadence = Duration::from_secs(30);
14782        for id in FLEET {
14783            assert_eq!(
14784                jittered_health_delay(id, 0, cadence),
14785                jittered_health_delay(id, 0, cadence),
14786                "{id}: the same module and probe index must produce the same offset"
14787            );
14788        }
14789    }
14790
14791    /// A zero cadence disables probing rather than producing a busy loop.
14792    #[test]
14793    fn zero_cadence_yields_zero_delay() {
14794        assert_eq!(
14795            jittered_health_delay("aft", 0, Duration::ZERO),
14796            Duration::ZERO
14797        );
14798    }
14799}
14800
14801#[cfg(all(test, target_os = "linux"))]
14802mod cgroup_placement_tests {
14803    use super::{
14804        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
14805        SuperviseError, SupervisedChild,
14806    };
14807    use crate::stderr_tail::{StderrRing, StderrTailConfig};
14808    use cortexkit_test_support::ScratchDir;
14809    use std::{
14810        fs, io,
14811        path::{Path, PathBuf},
14812        sync::{Arc, Mutex},
14813    };
14814    use tokio::process::Command;
14815
14816    #[tokio::test]
14817    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14818        use super::*;
14819        let dir = ScratchDir::new("unique-spawn-cgroups");
14820        let root = PathBuf::from(format!(
14821            "/sys/fs/cgroup/subc-unique-test-{}-{}",
14822            std::process::id(),
14823            unix_ms_now()
14824        ));
14825        if let Err(error) = fs::create_dir(&root) {
14826            assert!(
14827                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14828                "required cgroup test cannot execute: {error}"
14829            );
14830            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14831            return;
14832        }
14833        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14834        let group_count = || {
14835            fs::read_dir(root.join("subc-modules"))
14836                .unwrap()
14837                .map(|entry| entry.unwrap().file_type().unwrap())
14838                .filter(|kind| kind.is_dir())
14839                .count()
14840        };
14841        let supervisor =
14842            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14843                .with_cgroup_placement(Some(placement.clone()));
14844        let runtime = supervisor.runtime_config();
14845        let mut spec = ModuleSpec {
14846            module_id: "unique-spawn".into(),
14847            program: PathBuf::from("/bin/sleep"),
14848            args: vec!["60".into()],
14849            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14850                .into_iter()
14851                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14852                .collect(),
14853            reserved: false,
14854            reserved_prefixes: vec![],
14855            protocol: ModuleProtocol::None,
14856            overlap: Default::default(),
14857        };
14858        let spawn = |spec: &ModuleSpec| {
14859            spawn_child(
14860                spec,
14861                None,
14862                None,
14863                &runtime.stderr_ring,
14864                None,
14865                &runtime.child_roster,
14866                Some(&placement),
14867            )
14868            .unwrap()
14869        };
14870        let mut live = spawn(&spec);
14871        for _ in 0..3 {
14872            // A new process can enter the old slot while retirement is pending.
14873            let next = spawn(&spec);
14874            assert_ne!(live.module_id, next.module_id);
14875            live.start_kill().unwrap();
14876            live.wait().await.unwrap();
14877            live = next;
14878            assert!(
14879                live.child.try_wait().unwrap().is_none(),
14880                "retiring the old slot must not kill the replacement"
14881            );
14882            assert_eq!(
14883                group_count(),
14884                1,
14885                "only the live spawn's cgroup should remain"
14886            );
14887        }
14888        supervisor.begin_daemon_shutdown();
14889        let reap = tokio::spawn(async move {
14890            live.wait().await.unwrap();
14891        });
14892        supervisor
14893            .end_children_for_daemon_shutdown(false, std::future::pending())
14894            .await;
14895        reap.await.unwrap();
14896        assert_eq!(group_count(), 0);
14897        // A normal exit uses the same tree-cleanup path as a killed spawn.
14898        spec.program = PathBuf::from("/bin/true");
14899        spec.args.clear();
14900        let fresh_roster = ChildRoster::default();
14901        let mut short = spawn_child(
14902            &spec,
14903            None,
14904            None,
14905            &runtime.stderr_ring,
14906            None,
14907            &fresh_roster,
14908            Some(&placement),
14909        )
14910        .unwrap();
14911        short.wait().await.unwrap();
14912        assert_eq!(group_count(), 0);
14913        spec.module_id = "_".repeat(255);
14914        let mut long_id = spawn_child(
14915            &spec,
14916            None,
14917            None,
14918            &runtime.stderr_ring,
14919            None,
14920            &fresh_roster,
14921            Some(&placement),
14922        )
14923        .unwrap();
14924        long_id.wait().await.unwrap();
14925        assert_eq!(
14926            group_count(),
14927            0,
14928            "valid long module IDs must not exceed cgroup NAME_MAX"
14929        );
14930        fs::remove_dir(root.join("subc-modules")).unwrap();
14931        fs::remove_dir(root).unwrap();
14932        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14933    }
14934
14935    #[test]
14936    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14937        let path = Path::new("/definitely-missing-subc-cgroup");
14938        let mut command = Command::new("true");
14939        let error = apply_cgroup_placement(
14940            &mut command,
14941            &ModuleSpec {
14942                module_id: "broken-cgroup".to_string(),
14943                program: PathBuf::from("true"),
14944                args: Vec::new(),
14945                env: Vec::new(),
14946                reserved: false,
14947                reserved_prefixes: Vec::new(),
14948                protocol: ModuleProtocol::Subc,
14949                overlap: Default::default(),
14950            },
14951            path,
14952        )
14953        .expect_err("a parent cgroup open failure must reject the supervised spawn");
14954        let reason = error.to_string();
14955
14956        assert!(
14957            matches!(error, SuperviseError::Cgroup { .. }),
14958            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14959        );
14960        assert!(
14961            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14962            "parent cgroup open failure must name cgroup.procs: {reason}"
14963        );
14964    }
14965
14966    #[tokio::test]
14967    async fn reaping_a_child_removes_its_empty_module_cgroup() {
14968        let root = ScratchDir::new("supervisor-reap-cgroup");
14969        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14970        let placement = subc_cgroup::prepare_at(&root)
14971            .expect("prepare scratch cgroup root")
14972            .expect("scratch root has a cgroup.procs marker");
14973        let module_id = "reaped-module";
14974        let module = placement
14975            .module_path(module_id)
14976            .expect("create scratch module cgroup");
14977        let child = Command::new("true")
14978            .env("XDG_DATA_HOME", root.path())
14979            .env("XDG_RUNTIME_DIR", root.path())
14980            .env("XDG_CONFIG_HOME", root.path())
14981            .spawn()
14982            .expect("spawn short-lived child");
14983        let pid = child.id().expect("spawned child has pid");
14984        let mut child = SupervisedChild {
14985            child,
14986            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14987            protocol: ModuleProtocol::Subc,
14988            module_id: module_id.to_string(),
14989            cgroup_placement: Some(placement),
14990            stdout_pump: None,
14991            stderr_pump: None,
14992            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14993            spawned_at: tokio::time::Instant::now(),
14994            spawned_at_ms: 0,
14995            spawned_from: PathBuf::from("true"),
14996            launch_spec_digest: LaunchSpecDigest([0; 32]),
14997            spawned_file_identity: None,
14998            process_start_time: None,
14999            process_identity: None,
15000            pid,
15001            roster_guard: None,
15002            #[cfg(target_os = "macos")]
15003            privacy_exec: None,
15004            spawn_failure: None,
15005        };
15006
15007        child.wait().await.expect("reap short-lived child");
15008
15009        assert!(
15010            !module.exists(),
15011            "reaping the supervised child must remove its empty cgroup"
15012        );
15013    }
15014
15015    #[test]
15016    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15017        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15018        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15019        let placement = subc_cgroup::prepare_at(&root)
15020            .expect("prepare scratch cgroup root")
15021            .expect("scratch root has a cgroup.procs marker");
15022        let module = placement
15023            .module_path("surviving-module")
15024            .expect("create scratch module cgroup");
15025        fs::write(module.join("surviving-process"), b"still present")
15026            .expect("make scratch cgroup non-empty");
15027        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15028
15029        remove_module_cgroup(&placement, "surviving-module");
15030
15031        let logs = crate::router::test_log::captured_logs(&logs);
15032        assert!(
15033            module.exists(),
15034            "failed removal must leave the cgroup intact"
15035        );
15036        assert!(
15037            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15038                && logs.contains("surviving-module"),
15039            "best-effort removal must report the failure without returning it: {logs}"
15040        );
15041    }
15042
15043    #[test]
15044    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15045        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15046        let reason = SuperviseError::Spawn {
15047            program: PathBuf::from("/bin/true"),
15048            source: io::Error::from_raw_os_error(13),
15049            cgroup_path: Some(cgroup_path.clone()),
15050        }
15051        .to_string();
15052
15053        assert!(
15054            reason.contains(&cgroup_path.display().to_string()),
15055            "a pre_exec spawn failure must name the cgroup path: {reason}"
15056        );
15057    }
15058}
15059
15060#[cfg(test)]
15061mod spawn_subscriber_lag_tests {
15062    use super::*;
15063
15064    /// A subscriber whose connection stops draining is dropped once its frame
15065    /// channel fills. The client must learn that from a terminal Error frame
15066    /// after the frames already queued for it, not from a stream that simply
15067    /// goes quiet.
15068    #[tokio::test]
15069    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15070        let feed = SpawnEventFeed::default();
15071        feed.configure_incarnation("lag-incarnation".to_string());
15072        // A one-slot connection queue that nobody reads until the emits are
15073        // done: the forwarder parks on it and the subscriber channel fills.
15074        let (tx, mut rx) = mpsc::channel(1);
15075        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15076            .expect("subscribe");
15077        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15078        for index in 0..emitted {
15079            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15080            // Let the forwarder take what it can so the fill point is the
15081            // subscriber channel, not a scheduling accident.
15082            tokio::task::yield_now().await;
15083        }
15084        assert_eq!(
15085            feed.subscriber_count(),
15086            0,
15087            "the lagged subscriber must be removed"
15088        );
15089
15090        let mut data = Vec::new();
15091        let mut last = None;
15092        loop {
15093            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15094                .await
15095                .expect("the forwarder must finish once the subscriber is dropped");
15096            let Some(outbound) = next else { break };
15097            let frame = outbound.frame;
15098            if frame.header.ty == FrameType::StreamData {
15099                assert!(last.is_none(), "no data may follow the terminal frame");
15100                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15101                data.push(event.cursor.seq);
15102            } else {
15103                assert!(last.is_none(), "exactly one terminal frame");
15104                last = Some(frame);
15105            }
15106        }
15107        assert!(!data.is_empty(), "queued frames drain before the terminal");
15108        for pair in data.windows(2) {
15109            assert_eq!(
15110                pair[1],
15111                pair[0] + 1,
15112                "queued frames arrive dense and in order"
15113            );
15114        }
15115        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15116        assert_eq!(terminal.header.ty, FrameType::Error);
15117        assert_eq!(terminal.header.corr, 7);
15118        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15119        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15120        let detail = body.detail.expect("lagged error carries detail");
15121        assert_eq!(
15122            detail["first_undelivered_cursor"]["seq"],
15123            data.last().unwrap() + 1,
15124            "the named cursor is the first event the subscriber did not receive"
15125        );
15126        assert_eq!(
15127            detail["first_undelivered_cursor"]["daemon_incarnation"],
15128            "lag-incarnation"
15129        );
15130    }
15131}
15132
15133#[cfg(test)]
15134mod terminal_history_read_concurrency_tests {
15135    use super::*;
15136    use crate::terminal_journal::read_pause;
15137    use cortexkit_test_support::ScratchDir;
15138    use std::sync::mpsc as std_mpsc;
15139
15140    fn journaled_ring(
15141        journal: &Arc<crate::terminal_journal::TerminalJournal>,
15142    ) -> Arc<Mutex<TerminalRing>> {
15143        Arc::new(Mutex::new(
15144            TerminalRing::new(TerminalRingConfig::default(), 1)
15145                .with_journal(Some(Arc::clone(journal))),
15146        ))
15147    }
15148
15149    fn crash(at_ms: u64) -> ExitReport {
15150        ExitReport {
15151            kind: ExitKind::Crash,
15152            code: Some(1),
15153            signal: None,
15154            at_ms,
15155        }
15156    }
15157
15158    /// Record an exit on another thread and report whether it finished within
15159    /// `bound`. The recorder thread is left running if it did not.
15160    fn record_within(
15161        module_id: &'static str,
15162        ring: &Arc<Mutex<TerminalRing>>,
15163        at_ms: u64,
15164        bound: Duration,
15165    ) -> bool {
15166        let ring = Arc::clone(ring);
15167        let (done, done_rx) = std_mpsc::channel();
15168        std::thread::spawn(move || {
15169            record_terminal(
15170                module_id,
15171                &ring,
15172                &SpawnEventFeed::default(),
15173                &crash(at_ms),
15174                TerminalDisposition::Restarting,
15175            );
15176            let _ = done.send(());
15177        });
15178        done_rx.recv_timeout(bound).is_ok()
15179    }
15180
15181    /// A history read in progress must not hold the journal writer (which every
15182    /// module's exit recording needs) or the module's own ring. Exits recorded
15183    /// while the read is paused complete promptly; the paused read answers as of
15184    /// the moment it started, and the next read has each exit exactly once.
15185    #[test]
15186    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15187        let dir = ScratchDir::new("terminal-history-concurrent-read");
15188        let path = dir.join("terminals.jsonl");
15189        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15190            path.clone(),
15191            "daemon".into(),
15192        ));
15193        let reader_ring = journaled_ring(&journal);
15194        let other_ring = journaled_ring(&journal);
15195        assert!(record_within(
15196            "reader-module",
15197            &reader_ring,
15198            10,
15199            Duration::from_secs(5)
15200        ));
15201
15202        let (started, release) = read_pause::install(&path);
15203        let reading = {
15204            let ring = Arc::clone(&reader_ring);
15205            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15206        };
15207        started
15208            .recv_timeout(Duration::from_secs(5))
15209            .expect("the history read reached its pause");
15210
15211        let bound = Duration::from_secs(1);
15212        assert!(
15213            record_within("other-module", &other_ring, 20, bound),
15214            "another module's exit waited on a history read (journal writer held)"
15215        );
15216        assert!(
15217            record_within("reader-module", &reader_ring, 30, bound),
15218            "the read module's own exit waited on its history read (ring held)"
15219        );
15220
15221        drop(release);
15222        let paused = reading.join().unwrap();
15223        assert_eq!(
15224            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15225            vec![10],
15226            "an exit recorded after the read began lands in neither half of it"
15227        );
15228        assert_eq!(paused.journal_skipped_lines, 0);
15229        assert_eq!(paused.journal_read_errors, 0);
15230
15231        let after = durable_terminal_history_of(&reader_ring, "reader-module");
15232        assert_eq!(
15233            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15234            vec![10, 30],
15235            "the next read merges ring and journal with no duplicate"
15236        );
15237        assert_eq!(after.journal_skipped_lines, 0);
15238    }
15239}
15240
15241/// What a restart does with the exited process's output readers. These drive
15242/// the same settle functions the supervisor calls, with a reader the test
15243/// holds, so a reader that has not been scheduled by the bound is a controlled
15244/// input rather than something only a loaded machine produces.
15245#[cfg(test)]
15246mod stderr_settle_tests {
15247    use std::{
15248        future::Future,
15249        io,
15250        pin::Pin,
15251        sync::{Arc, Mutex},
15252        task::{Context, Poll},
15253        time::Duration,
15254    };
15255
15256    use tokio::{
15257        io::{AsyncRead, ReadBuf},
15258        sync::oneshot,
15259        time::Instant,
15260    };
15261
15262    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15263    use crate::stderr_tail::{
15264        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15265        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15266    };
15267
15268    const BOUND: Duration = Duration::from_millis(250);
15269
15270    /// Yields `before`, then stays pending until the gate is released, then
15271    /// yields `after` and reaches EOF. The bytes after the gate were written
15272    /// by a process that has already exited; only the reader is behind.
15273    struct HeldReader {
15274        before: Option<Vec<u8>>,
15275        gate: Option<oneshot::Receiver<()>>,
15276        after: io::Cursor<Vec<u8>>,
15277    }
15278
15279    impl AsyncRead for HeldReader {
15280        fn poll_read(
15281            mut self: Pin<&mut Self>,
15282            cx: &mut Context<'_>,
15283            buf: &mut ReadBuf<'_>,
15284        ) -> Poll<io::Result<()>> {
15285            if let Some(bytes) = self.before.take() {
15286                buf.put_slice(&bytes);
15287                return Poll::Ready(Ok(()));
15288            }
15289            if let Some(gate) = self.gate.as_mut() {
15290                match Pin::new(gate).poll(cx) {
15291                    Poll::Pending => return Poll::Pending,
15292                    Poll::Ready(_) => self.gate = None,
15293                }
15294            }
15295            Pin::new(&mut self.after).poll_read(cx, buf)
15296        }
15297    }
15298
15299    struct DiscardSink;
15300
15301    impl OutputSink for DiscardSink {
15302        fn write_line(&mut self, _line: &[u8]) {}
15303    }
15304
15305    fn line(text: &str) -> TailEntry {
15306        TailEntry::Line {
15307            text: text.to_string(),
15308            truncated: false,
15309            at_ms: None,
15310        }
15311    }
15312
15313    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15314        ring.lock().unwrap()
15315    }
15316
15317    /// Start a reader for a new process generation that delivers `before`
15318    /// immediately and `after` only once the returned sender fires (or is
15319    /// dropped).
15320    fn held_pump(
15321        ring: &Arc<Mutex<StderrRing>>,
15322        before: &str,
15323        after: &str,
15324    ) -> (StderrPump, oneshot::Sender<()>) {
15325        let generation = lock(ring).begin_process();
15326        let (release, gate) = oneshot::channel();
15327        let reader = HeldReader {
15328            before: Some(before.as_bytes().to_vec()),
15329            gate: Some(gate),
15330            after: io::Cursor::new(after.as_bytes().to_vec()),
15331        };
15332        let task = tokio::spawn(pump_stderr_to(
15333            reader,
15334            Arc::clone(ring),
15335            generation,
15336            DiscardSink,
15337        ));
15338        (StderrPump { task, generation }, release)
15339    }
15340
15341    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15342        for _ in 0..1000 {
15343            if done(&lock(ring)) {
15344                return;
15345            }
15346            tokio::time::sleep(Duration::from_millis(1)).await;
15347        }
15348        panic!(
15349            "ring never reached the expected state: {:?}",
15350            lock(ring).snapshot(None, None)
15351        );
15352    }
15353
15354    #[tokio::test(start_paused = true)]
15355    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15356        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15357        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15358
15359        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15360        let before_release = lock(&ring).snapshot(None, None);
15361        assert!(
15362            matches!(before_release.capture, CaptureState::Incomplete { .. }),
15363            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15364        );
15365
15366        // The restart: the next process starts and writes before the old
15367        // reader catches up.
15368        let next = lock(&ring).begin_process();
15369        lock(&ring).push_line_from(next, "next process booting");
15370        release.send(()).unwrap();
15371        wait_until(&ring, |ring| {
15372            ring.snapshot(None, None).capture == CaptureState::Captured
15373        })
15374        .await;
15375
15376        assert_eq!(
15377            untimed(lock(&ring).snapshot(None, None).entries),
15378            vec![
15379                line("booting"),
15380                line("config error: missing storage"),
15381                TailEntry::ProcessStart,
15382                line("next process booting"),
15383            ],
15384            "the crash's last line must survive a slow reader and stay in the crashed process's section"
15385        );
15386    }
15387
15388    #[tokio::test(start_paused = true)]
15389    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15390    ) {
15391        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15392        // `_held` is never fired: a descendant keeps the pipe open for the
15393        // whole test.
15394        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15395
15396        let started = Instant::now();
15397        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15398        assert_eq!(
15399            started.elapsed(),
15400            BOUND,
15401            "the restart must wait exactly the bound for a pipe that stays open, no longer"
15402        );
15403
15404        let next = lock(&ring).begin_process();
15405        lock(&ring).push_line_from(next, "next process booting");
15406        tokio::time::sleep(Duration::from_secs(60)).await;
15407
15408        let snapshot = lock(&ring).snapshot(None, None);
15409        match &snapshot.capture {
15410            CaptureState::Incomplete { reason } => assert!(
15411                reason.contains("had not reached EOF") && reason.contains("250ms"),
15412                "the reason must say what is missing and after how long: {reason}"
15413            ),
15414            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15415        }
15416        assert_eq!(
15417            untimed(snapshot.entries),
15418            vec![
15419                line("parent exiting"),
15420                TailEntry::ProcessStart,
15421                line("next process booting"),
15422            ]
15423        );
15424    }
15425
15426    #[tokio::test(start_paused = true)]
15427    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15428        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15429        let (pump, release) = held_pump(&ring, "one\n", "two\n");
15430        release.send(()).unwrap();
15431
15432        settle_stderr_pump("clean", &ring, pump, BOUND).await;
15433
15434        let snapshot = lock(&ring).snapshot(None, None);
15435        assert_eq!(snapshot.capture, CaptureState::Captured);
15436        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15437    }
15438
15439    #[tokio::test(start_paused = true)]
15440    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15441        const BURST: usize = 150;
15442        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15443        let path = capture.join("two-pipe.stderr.log");
15444        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15445        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15446        let generation = lock(&ring).begin_process();
15447        let stdout = (0..BURST)
15448            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15449            .collect::<String>();
15450        let stderr = (0..BURST)
15451            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15452            .collect::<String>();
15453        // Model a child that has written both bursts and exited. Hold the stdout
15454        // reader after one line and a partial prefix, as if pipe readiness or
15455        // capture-file writes had delayed it. The final line also needs EOF to
15456        // flush: neither complete nor partial buffered output may be discarded.
15457        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15458        let (release, gate) = oneshot::channel();
15459        let reader = HeldReader {
15460            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15461            gate: Some(gate),
15462            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15463        };
15464        let (finished, completion) = oneshot::channel();
15465        let stdout_sink = sink.clone();
15466        let pump = tokio::spawn(async move {
15467            pump_stdout_to(reader, stdout_sink).await;
15468            finished.send(()).unwrap();
15469        });
15470        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15471
15472        let started = Instant::now();
15473        settle_stdout_pump("two-pipe", pump, BOUND).await;
15474        assert_eq!(
15475            started.elapsed(),
15476            BOUND,
15477            "a late reader must not hold up restart"
15478        );
15479        assert_eq!(
15480            std::fs::read_to_string(&path).unwrap().lines().count(),
15481            BURST + 1
15482        );
15483
15484        release
15485            .send(())
15486            .expect("stdout reader must survive the exit drain bound");
15487        tokio::time::timeout(Duration::from_secs(1), completion)
15488            .await
15489            .expect("released stdout reader must reach EOF")
15490            .expect("stdout capture must finish at EOF, not be cancelled");
15491        let contents = std::fs::read_to_string(&path).unwrap();
15492        let mut lines = contents
15493            .lines()
15494            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15495            .collect::<Vec<_>>();
15496        let mut expected = (0..BURST)
15497            .flat_map(|i| {
15498                [
15499                    format!("out-{i:04}-{}", "o".repeat(64)),
15500                    format!("err-{i:04}-{}", "e".repeat(64)),
15501                ]
15502            })
15503            .collect::<Vec<_>>();
15504        lines.sort_unstable();
15505        expected.sort_unstable();
15506        assert_eq!(
15507            lines, expected,
15508            "both pipes must retain every complete line exactly once"
15509        );
15510    }
15511}
15512
15513/// Containment of a module's process tree (issue #109).
15514///
15515/// The behaviour these defend against is a module helper surviving its module:
15516/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
15517/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
15518/// compounds it.
15519///
15520/// They run against the SUPERVISOR rather than the job-object crate because the
15521/// claim is about teardown: a crate-level test proves a job can reap a tree, not
15522/// that the daemon's drain path reaches it.
15523///
15524/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
15525/// lane there is a separate containment path with its own tests.
15526#[cfg(all(test, windows))]
15527mod job_containment_tests {
15528    use super::*;
15529    use cortexkit_test_support::ScratchDir;
15530    use std::{
15531        path::{Path, PathBuf},
15532        sync::{Arc, Mutex},
15533        time::{Duration, Instant},
15534    };
15535
15536    /// The stub, expected beside this test executable.
15537    ///
15538    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
15539    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
15540    /// failure then reads as a broken test rather than an unbuilt dependency.
15541    fn stub_path() -> PathBuf {
15542        let mut path = std::env::current_exe().expect("current_exe available in tests");
15543        path.pop();
15544        path.pop();
15545        path.push("fake-aft-stub.exe");
15546        assert!(
15547            path.exists(),
15548            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15549             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15550            path.display()
15551        );
15552        path
15553    }
15554
15555    /// Poll for the grandchild pid the stub records, and parse it.
15556    fn read_grandchild_pid(path: &Path) -> u32 {
15557        let deadline = Instant::now() + Duration::from_secs(10);
15558        loop {
15559            if let Ok(contents) = std::fs::read_to_string(path) {
15560                if let Ok(pid) = contents.trim().parse() {
15561                    return pid;
15562                }
15563            }
15564            assert!(
15565                Instant::now() < deadline,
15566                "the stub never recorded a grandchild pid at {}",
15567                path.display()
15568            );
15569            std::thread::sleep(Duration::from_millis(10));
15570        }
15571    }
15572
15573    /// Everything one fixture run needs, so the two tests below differ in exactly
15574    /// one place: whether the child is contained.
15575    struct Fixture {
15576        _dir: ScratchDir,
15577        module_id: String,
15578        grandchild: u32,
15579        child: Option<SupervisedChild>,
15580        registry: Arc<Registry>,
15581        snapshot: Arc<Mutex<SupervisorSnapshot>>,
15582        terminal_ring: Arc<Mutex<TerminalRing>>,
15583        spawn_events: SpawnEventFeed,
15584    }
15585
15586    fn fixture(label: &str, module_id: &str) -> Fixture {
15587        let dir = ScratchDir::new(label);
15588        let pid_file = dir.join("grandchild.pid");
15589        let supervisor = Supervisor::new_for_test(
15590            Arc::new(Registry::default()),
15591            RestartPolicy::new(3, Duration::ZERO),
15592        );
15593        let runtime = supervisor.runtime_config();
15594        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15595        let spec = ModuleSpec {
15596            module_id: module_id.to_string(),
15597            program: stub_path(),
15598            // Zero args deliberately: a `--subc` argument would make the stub dial
15599            // a daemon that is not there, and the failure would land in the same
15600            // stderr ring this fixture exists to keep quiet.
15601            args: Vec::new(),
15602            env: vec![
15603                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15604                (
15605                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15606                    pid_file.display().to_string(),
15607                ),
15608            ],
15609            reserved: false,
15610            reserved_prefixes: Vec::new(),
15611            protocol: ModuleProtocol::Subc,
15612            overlap: Default::default(),
15613        };
15614        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15615            .expect("spawn the supervised fixture");
15616        let grandchild = read_grandchild_pid(&pid_file);
15617        Fixture {
15618            _dir: dir,
15619            module_id: module_id.to_string(),
15620            grandchild,
15621            child: Some(child),
15622            registry: Arc::new(Registry::default()),
15623            snapshot,
15624            terminal_ring: Arc::clone(&runtime.terminal_ring),
15625            spawn_events: SpawnEventFeed::default(),
15626        }
15627    }
15628
15629    impl Fixture {
15630        /// Drain through the supervisor's own teardown path.
15631        async fn drain(&mut self) {
15632            let child = self
15633                .child
15634                .take()
15635                .expect("the fixture child is still present");
15636            drain_child_to_state(
15637                &self.module_id,
15638                ModuleProtocol::Subc,
15639                // No forwarding table in this fixture, so nothing reaches the
15640                // child over a connection.
15641                StopNotice::NotSent,
15642                &self.registry,
15643                None,
15644                &self.snapshot,
15645                &self.terminal_ring,
15646                &self.spawn_events,
15647                child,
15648                Duration::from_millis(500),
15649                ModuleState::Stopped,
15650                Some(false),
15651            )
15652            .await
15653            .expect("drain the supervised fixture");
15654        }
15655    }
15656
15657    /// Teardown reaps the grandchild, not merely the direct child.
15658    ///
15659    /// This is the assertion the change exists for. Before containment the
15660    /// grandchild survived: it is a separate process, and `start_kill` is
15661    /// `TerminateProcess` scoped to one pid.
15662    #[tokio::test]
15663    async fn teardown_reaps_the_grandchild() {
15664        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15665        let grandchild = fixture.grandchild;
15666
15667        assert!(
15668            subc_jobobject::process_exists(grandchild),
15669            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15670        );
15671
15672        fixture.drain().await;
15673
15674        assert!(
15675            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15676            "grandchild {grandchild} outlived module teardown: the tree was not contained"
15677        );
15678    }
15679
15680    /// The mutation control: with containment withheld, the grandchild survives
15681    /// the same kill.
15682    ///
15683    /// This is the defect reproduction from #109 — a direct-child kill reaches
15684    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
15685    /// supervisor because `spawn_and_mark_running` now always contains on
15686    /// Windows, which is the point: there is no longer a path that spawns
15687    /// uncontained, so the control has to construct one.
15688    ///
15689    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
15690    /// grandchild ever dies here, that test is passing for a reason unrelated to
15691    /// the job object and the containment claim is unproven.
15692    #[test]
15693    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15694        let dir = ScratchDir::new("teardown-uncontained");
15695        let pid_file = dir.join("grandchild.pid");
15696        let mut child = std::process::Command::new(stub_path())
15697            .env("FAKE_AFT_NEVER_CONNECT", "1")
15698            .env(
15699                "FAKE_AFT_GRANDCHILD_PID_FILE",
15700                pid_file.display().to_string(),
15701            )
15702            .stdin(std::process::Stdio::null())
15703            .stdout(std::process::Stdio::null())
15704            .stderr(std::process::Stdio::null())
15705            .spawn()
15706            .expect("spawn the uncontained fixture");
15707        let grandchild = read_grandchild_pid(&pid_file);
15708
15709        // Exactly what the pre-fix teardown did: kill the direct child.
15710        child.kill().expect("kill the direct child");
15711        let _ = child.wait();
15712
15713        assert!(
15714            subc_jobobject::process_exists(grandchild),
15715            "grandchild {grandchild} died with the direct child, so this control no longer \
15716             distinguishes contained from uncontained teardown and the regression test is \
15717             passing vacuously"
15718        );
15719
15720        // The orphan this control demonstrates is the leak the fix prevents, so
15721        // the control must not leave one behind.
15722        kill_tree(grandchild);
15723    }
15724
15725    /// Crash durability: closing the containment handle reaps the tree with no
15726    /// teardown code running at all.
15727    ///
15728    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
15729    /// call anything — and it is why containment is a kernel property of the
15730    /// handle rather than a step in the drain. Discovered by getting the
15731    /// mutation control wrong: clearing `job` to "disable" containment instead
15732    /// killed the tree, which is the guarantee, not a mistake.
15733    #[tokio::test]
15734    async fn dropping_containment_reaps_the_grandchild() {
15735        let mut fixture = fixture("drop-containment", "tree-drop");
15736        let grandchild = fixture.grandchild;
15737
15738        assert!(subc_jobobject::process_exists(grandchild));
15739
15740        // No `drain` call, no kill: dropping the handle is the entire mechanism.
15741        fixture.child.as_mut().expect("child present").job = None;
15742
15743        assert!(
15744            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15745            "grandchild {grandchild} survived the containment handle closing, so a daemon \
15746             crash would leave the tree behind"
15747        );
15748    }
15749
15750    /// Kill a pid and its tree, then confirm it is gone.
15751    fn kill_tree(pid: u32) {
15752        let _ = std::process::Command::new("taskkill.exe")
15753            .args(["/PID", &pid.to_string(), "/T", "/F"])
15754            .stdin(std::process::Stdio::null())
15755            .stdout(std::process::Stdio::null())
15756            .stderr(std::process::Stdio::null())
15757            .status();
15758        assert!(
15759            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15760            "could not clean up grandchild {pid}"
15761        );
15762    }
15763}
15764
15765#[cfg(test)]
15766mod privacy_trampoline_configuration_tests {
15767    #[tokio::test]
15768    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15769    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15770        #[cfg(target_os = "macos")]
15771        {
15772            let supervisor = super::Supervisor::new(
15773                std::sync::Arc::new(crate::Registry::default()),
15774                super::RestartPolicy::default(),
15775            );
15776            let error = supervisor.spawn(spec()).unwrap_err();
15777            assert!(
15778                error
15779                    .to_string()
15780                    .contains("no privacy trampoline configured"),
15781                "{error}"
15782            );
15783        }
15784    }
15785
15786    #[tokio::test]
15787    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15788    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15789        #[cfg(target_os = "macos")]
15790        {
15791            let supervisor = super::Supervisor::new(
15792                std::sync::Arc::new(crate::Registry::default()),
15793                super::RestartPolicy::default(),
15794            )
15795            .with_privacy_trampoline(std::env::current_exe().unwrap());
15796            let error = supervisor.spawn(spec()).unwrap_err();
15797            assert!(
15798                error
15799                    .to_string()
15800                    .contains("binary does not implement the privacy trampoline protocol"),
15801                "{error}"
15802            );
15803        }
15804    }
15805
15806    #[cfg(target_os = "macos")]
15807    fn spec() -> super::ModuleSpec {
15808        super::ModuleSpec {
15809            module_id: "privacy-configuration".into(),
15810            program: "/bin/sleep".into(),
15811            args: vec!["30".into()],
15812            env: vec![],
15813            reserved: false,
15814            reserved_prefixes: vec![],
15815            protocol: subc_control::ModuleProtocol::None,
15816            overlap: super::ModuleOverlap::Exclusive,
15817        }
15818    }
15819}
15820
15821#[cfg(test)]
15822mod privacy_exec_boundary_tests {
15823    #[cfg(target_os = "macos")]
15824    use super::*;
15825    #[cfg(target_os = "macos")]
15826    use std::{
15827        io::{Read, Write},
15828        net::{TcpListener, TcpStream},
15829    };
15830
15831    /// Unit-test-only pause at the actual early image read, not at a later
15832    /// status read. Production supervisors never inspect this environment key.
15833    #[cfg(target_os = "macos")]
15834    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15835        if let Some((_, path)) = spec
15836            .env
15837            .iter()
15838            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15839        {
15840            let mut barrier = TcpStream::connect(path).unwrap();
15841            barrier
15842                .set_read_timeout(Some(Duration::from_secs(30)))
15843                .unwrap();
15844            barrier.write_all(&pid.to_ne_bytes()).unwrap();
15845            let mut release = [0];
15846            barrier.read_exact(&mut release).unwrap();
15847            assert_eq!(&release, b"X");
15848        }
15849    }
15850
15851    #[cfg(target_os = "macos")]
15852    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15853        ModuleSpec {
15854            module_id: "privacy-boundary".into(),
15855            program: program.into(),
15856            args: args.iter().map(|arg| (*arg).into()).collect(),
15857            env: vec![],
15858            reserved: false,
15859            reserved_prefixes: vec![],
15860            protocol: ModuleProtocol::None,
15861            overlap: ModuleOverlap::Exclusive,
15862        }
15863    }
15864
15865    #[cfg(target_os = "macos")]
15866    async fn accept(listener: TcpListener) -> TcpStream {
15867        // Socket readiness, not elapsed time, establishes both pause points.
15868        let listener = tokio::net::TcpListener::from_std({
15869            listener.set_nonblocking(true).unwrap();
15870            listener
15871        })
15872        .unwrap();
15873        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15874            .await
15875            .unwrap()
15876            .unwrap();
15877        let stream = stream.into_std().unwrap();
15878        stream.set_nonblocking(false).unwrap();
15879        stream
15880            .set_read_timeout(Some(Duration::from_secs(30)))
15881            .unwrap();
15882        stream
15883    }
15884
15885    #[tokio::test]
15886    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15887    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15888        #[cfg(target_os = "macos")]
15889        {
15890            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15891            // Loopback sockets also work when the replay adapter's TMPDIR is
15892            // longer than Darwin's Unix-domain socket path limit.
15893            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15894            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15895            let record = root.join("live-children.json");
15896            let supervisor =
15897                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15898                    .with_live_children_record(&record);
15899            let runtime = supervisor.runtime_config();
15900            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15901            let mut spec = spec("/bin/sleep", &["30"]);
15902            spec.env = vec![
15903                (
15904                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15905                    exec_listener.local_addr().unwrap().to_string(),
15906                ),
15907                (
15908                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15909                    sample_listener.local_addr().unwrap().to_string(),
15910                ),
15911            ];
15912            let spawn = tokio::task::spawn_blocking(move || {
15913                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15914            });
15915            let mut sample = accept(sample_listener).await;
15916            let mut pid = [0; 4];
15917            sample.read_exact(&mut pid).unwrap();
15918            let pid = u32::from_ne_bytes(pid);
15919            let mut exec = accept(exec_listener).await;
15920            let mut ready = [0];
15921            exec.read_exact(&mut ready).unwrap();
15922            assert_eq!(&ready, b"R");
15923            // The early read is guaranteed to see a real, non-null trampoline
15924            // image: the fixture has reached its barrier and cannot exec yet.
15925            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15926            assert_eq!(
15927                observe_spawned_image(pid).unwrap().executable,
15928                Some(trampoline)
15929            );
15930            sample.write_all(b"X").unwrap();
15931            let mut child = spawn.await.unwrap();
15932            let early = crate::live_children::read_record(&record).unwrap();
15933            assert_eq!(early.len(), 1);
15934            assert_eq!(early[0].pid, pid);
15935            assert_eq!(
15936                early[0].executable, None,
15937                "unconfirmed trampoline image entered the roster"
15938            );
15939            assert!(child.report_ready.get().is_none());
15940            // The barrier's duration is unrelated to the production five-second
15941            // exec budget. Start the test's confirmation budget upon release.
15942            child.privacy_exec.as_mut().unwrap().deadline =
15943                tokio::time::Instant::now() + Duration::from_secs(30);
15944            exec.write_all(b"X").unwrap();
15945            child.confirm_privacy_exec().await;
15946            assert_eq!(child.spawn_failure, None);
15947            assert!(child.report_ready.get().is_some());
15948            let confirmed = crate::live_children::read_record(&record).unwrap();
15949            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15950            assert_ne!(module, trampoline);
15951            assert_eq!(confirmed[0].executable, Some(module.into()));
15952            child.start_kill().unwrap();
15953            child.wait().await.unwrap();
15954            child.release_roster();
15955        }
15956    }
15957
15958    #[tokio::test]
15959    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15960    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15961        #[cfg(target_os = "macos")]
15962        {
15963            let registry = Arc::new(Registry::default());
15964            let policy = RestartPolicy::new(0, Duration::ZERO);
15965            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
15966            let runtime = supervisor.runtime_config();
15967            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15968            let spec = spec("/bin/sh", &["-c", "exit 121"]);
15969            // Drive spawn and confirmation separately instead of starting the
15970            // monitor. WNOWAIT observes a real exit without consuming its status,
15971            // so confirmation's first try_wait must take the already-exited arm.
15972            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15973            let pid = child.pid;
15974            tokio::task::spawn_blocking(move || {
15975                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15976            })
15977            .await
15978            .unwrap()
15979            .unwrap();
15980            child.privacy_exec.as_mut().unwrap().deadline =
15981                tokio::time::Instant::now() + Duration::from_secs(30);
15982            let status = child.wait().await.unwrap();
15983            assert_eq!(status.code(), Some(121));
15984            assert!(child.privacy_exec.is_none());
15985            assert!(
15986                child.report_ready.get().is_none(),
15987                "an exited module must not publish a live pid"
15988            );
15989            let report = classify_reaped_child_exit(&snapshot, &child, &status);
15990            on_child_exit(
15991                &spec,
15992                policy,
15993                &registry,
15994                &snapshot,
15995                &runtime.terminal_ring,
15996                &runtime.spawn_events,
15997                &runtime.child_roster,
15998                report,
15999            )
16000            .await;
16001            let state = lock_snapshot(&snapshot).unwrap();
16002            assert_eq!(state.state, ModuleState::Failed);
16003            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16004            assert_eq!(state.reported_pid(), None);
16005            drop(state);
16006            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16007            assert_eq!(history.entries.len(), 1);
16008            let terminal = &history.entries[0];
16009            assert_eq!(terminal.exit_code, Some(121));
16010            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16011            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16012            assert_eq!(
16013                terminal.disposition_detail.as_deref(),
16014                Some(policy.budget_exhausted_detail().as_str()),
16015                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16016            );
16017            assert_eq!(child.spawn_failure, None);
16018            child.release_roster();
16019        }
16020    }
16021}
16022
16023/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16024/// descriptor 3, without an environment copy. The shell records the nonce
16025/// and its environment after exec so these tests observe the real handover.
16026#[cfg(all(test, unix))]
16027mod launch_nonce_descriptor_tests {
16028    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16029    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16030    use cortexkit_test_support::ScratchDir;
16031    use std::{
16032        path::PathBuf,
16033        sync::{Arc, Mutex},
16034        time::{Duration, Instant},
16035    };
16036
16037    async fn probe(role: super::SpawnRole) {
16038        let scratch = ScratchDir::new("launch-nonce-descriptor");
16039        let fd_copy = scratch.join("from-descriptor");
16040        let env_copy = scratch.join("environment");
16041        let script = format!(
16042            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16043            fd = fd_copy.display(), env = env_copy.display(),
16044        );
16045        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16046        let spec = ModuleSpec {
16047            module_id: "nonce-descriptor-probe".to_string(),
16048            program: PathBuf::from("/bin/sh"),
16049            args: vec!["-c".to_string(), script],
16050            env: vec![
16051                xdg("XDG_DATA_HOME"),
16052                xdg("XDG_RUNTIME_DIR"),
16053                xdg("XDG_CONFIG_HOME"),
16054                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16055            ],
16056            reserved: true,
16057            reserved_prefixes: Vec::new(),
16058            protocol: ModuleProtocol::Subc,
16059            overlap: Default::default(),
16060        };
16061        let handle = SupervisorHandle::new();
16062        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16063        let roster = ChildRoster::default();
16064        #[cfg(target_os = "macos")]
16065        {
16066            let path = super::test_privacy_trampoline();
16067            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16068        }
16069        let child = super::spawn_child_in_slot(
16070            &spec,
16071            None,
16072            Some(&handle),
16073            &ring,
16074            None,
16075            &roster,
16076            #[cfg(target_os = "linux")]
16077            None,
16078            role,
16079            matches!(role, super::SpawnRole::SwapCandidate),
16080        )
16081        .expect("spawn probe");
16082        let deadline = Instant::now() + Duration::from_secs(10);
16083        while !(fd_copy.exists() && env_copy.exists()) {
16084            assert!(Instant::now() < deadline, "probe never wrote its copies");
16085            tokio::time::sleep(Duration::from_millis(20)).await;
16086        }
16087        let nonce = std::fs::read_to_string(fd_copy).unwrap();
16088        assert!(!nonce.is_empty());
16089        let environment = std::fs::read_to_string(env_copy).unwrap();
16090        assert!(environment
16091            .lines()
16092            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16093        let copy = environment
16094            .lines()
16095            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16096        assert_eq!(
16097            copy, None,
16098            "Unix children must never receive the environment nonce"
16099        );
16100        if matches!(role, super::SpawnRole::Plain) {
16101            assert_eq!(
16102                handle.spawn_nonce(&spec.module_id).as_deref(),
16103                Some(nonce.as_str())
16104            );
16105        }
16106        drop(child);
16107    }
16108
16109    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16110    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16111        probe(super::SpawnRole::Plain).await;
16112    }
16113
16114    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16115    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16116        probe(super::SpawnRole::SwapCandidate).await;
16117    }
16118}
16119
16120#[cfg(all(test, target_os = "linux"))]
16121mod cgroup_containment_tests {
16122    use super::*;
16123    use cortexkit_test_support::ScratchDir;
16124
16125    fn running(pid: u32) -> bool {
16126        // An orphan can remain a zombie until the container init reaps it.
16127        std::fs::read_to_string(format!("/proc/{pid}/stat"))
16128            .ok()
16129            .and_then(|stat| {
16130                stat.rsplit_once(") ")
16131                    .map(|(_, rest)| rest.starts_with('Z'))
16132            })
16133            .is_some_and(|zombie| !zombie)
16134    }
16135
16136    #[tokio::test]
16137    async fn linux_teardown_reaps_the_grandchild() {
16138        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16139    }
16140
16141    #[tokio::test]
16142    async fn linux_shutdown_straggler_reaps_the_grandchild() {
16143        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16144    }
16145
16146    async fn teardown_tree(test_name: &str, shutdown: bool) {
16147        let dir = ScratchDir::new(test_name);
16148        let root = PathBuf::from(format!(
16149            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16150            std::process::id(),
16151            unix_ms_now()
16152        ));
16153        if let Err(error) = std::fs::create_dir(&root) {
16154            assert!(
16155                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16156                "required cgroup test cannot execute: {error}"
16157            );
16158            eprintln!(
16159                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16160                root.display()
16161            );
16162            return;
16163        }
16164        let placement = subc_cgroup::prepare_at(&root)
16165            .expect("prepare isolated kernel cgroup")
16166            .expect("isolated cgroup is delegated");
16167        let module_id = "tree-teardown";
16168        let module = placement
16169            .module_path(module_id)
16170            .expect("create isolated module cgroup");
16171        if !module.join("cgroup.kill").exists() {
16172            std::fs::remove_dir(&module).unwrap();
16173            std::fs::remove_dir(root.join("subc-modules")).unwrap();
16174            std::fs::remove_dir(&root).unwrap();
16175            assert!(
16176                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16177                "required cgroup.kill interface unavailable"
16178            );
16179            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16180            return;
16181        }
16182        let supervisor = Supervisor::new_for_test(
16183            Arc::new(Registry::default()),
16184            RestartPolicy::new(3, Duration::ZERO),
16185        )
16186        .with_cgroup_placement(Some(placement));
16187        let mut runtime = supervisor.runtime_config();
16188        runtime.child_roster = runtime
16189            .child_roster
16190            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16191        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16192        let pid_file = dir.join("grandchild.pid");
16193        let spec = ModuleSpec {
16194            module_id: module_id.to_string(),
16195            program: PathBuf::from("/bin/sh"),
16196            args: vec![
16197                "-c".into(),
16198                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16199                "fixture".into(),
16200                pid_file.display().to_string(),
16201            ],
16202            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16203                .into_iter()
16204                .map(|key| (key.to_string(), dir.display().to_string()))
16205                .collect(),
16206            reserved: false,
16207            reserved_prefixes: Vec::new(),
16208            protocol: ModuleProtocol::None,
16209            overlap: Default::default(),
16210        };
16211        let child =
16212            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16213        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16214        let grandchild: u32 = loop {
16215            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16216                if let Ok(pid) = contents.trim().parse() {
16217                    break pid;
16218                }
16219            }
16220            assert!(
16221                tokio::time::Instant::now() < deadline,
16222                "grandchild pid was not recorded"
16223            );
16224            tokio::time::sleep(Duration::from_millis(10)).await;
16225        };
16226        assert!(
16227            running(grandchild),
16228            "grandchild must be alive before teardown"
16229        );
16230        if shutdown {
16231            let mut child = child;
16232            crate::child_roster::end_children_for_daemon_shutdown(
16233                &runtime.child_roster,
16234                false,
16235                std::future::pending(),
16236            )
16237            .await;
16238            child.wait().await.expect("reap shutdown straggler");
16239        } else {
16240            drain_child_to_state(
16241                module_id,
16242                ModuleProtocol::None,
16243                StopNotice::NotSent,
16244                &Registry::default(),
16245                None,
16246                &snapshot,
16247                &runtime.terminal_ring,
16248                &SpawnEventFeed::default(),
16249                child,
16250                Duration::from_millis(100),
16251                ModuleState::Stopped,
16252                Some(false),
16253            )
16254            .await
16255            .expect("real supervisor teardown");
16256        }
16257        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16258        while running(grandchild) && tokio::time::Instant::now() < deadline {
16259            tokio::time::sleep(Duration::from_millis(10)).await;
16260        }
16261        let survived = running(grandchild);
16262        // Kill a surviving grandchild so a failed test does not leave it behind.
16263        if survived {
16264            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16265            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16266            tokio::time::sleep(Duration::from_millis(100)).await;
16267        }
16268        if module.exists() {
16269            std::fs::remove_dir(&module).expect("remove empty module cgroup");
16270        }
16271        std::fs::remove_dir(root.join("subc-modules")).unwrap();
16272        std::fs::remove_dir(&root).unwrap();
16273        assert!(
16274            !survived,
16275            "grandchild {grandchild} outlived module teardown"
16276        );
16277        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16278    }
16279}