Skip to main content

subc_daemon/
supervise.rs

1use std::{
2    collections::{HashMap, HashSet, VecDeque},
3    error::Error,
4    fmt, io,
5    path::PathBuf,
6    process::{ExitStatus, Stdio},
7    sync::{Arc, Mutex, OnceLock},
8    time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15    ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16    SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19    manifest::{SelfSignalKind, SignalAnchor},
20    session::{
21        HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22        MODULE_CONTROL_OP_HEALTH_CHECK,
23    },
24    Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27    process::{Child, Command},
28    sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29    task::JoinHandle,
30    time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35    child_roster::ChildRoster,
36    daemon_config::{
37        CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38    },
39    forwarding::{
40        CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41        ModuleDrainTarget, PendingModuleControlRpc,
42    },
43    provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44    registry::{ConnectionId, RegistrationEndReason, RegistryError},
45    stderr_tail::{
46        pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47        StderrTailSnapshot,
48    },
49    terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50    Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62/// Command-line flag used by supervised modules to find subc.
63///
64/// subc launches module-mode children as `<module> --subc <connection-file-path>`.
65/// The path points at the TCP+key connection file; it is not an ambient signal and
66/// is never inherited by standalone children.
67pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72/// The span `DEFAULT_MAX_RESTARTS` is counted over. Ten minutes is long enough
73/// to contain a real crash loop (which respawns in seconds) and short enough
74/// that unrelated crashes hours apart never accumulate into a permanent stop.
75const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76/// How long a drain waits for already-dispatched requests to finalize before
77/// the child is torn down. Sized for TOOL-SCALE work (bash, inspect, builds),
78/// not RPC-scale: the original 2s value silently cut nearly every real tool
79/// call at the fence, making the wait-for-finalize design decorative for the
80/// workloads it existed for. Quiescence short-circuits, so an idle module
81/// restarts immediately regardless of this value; the budget is spent only
82/// when a genuine in-flight request is worth finishing. Per-module override:
83/// `drain_timeout_ms` in subc.jsonc; per-restart override: the operator's
84/// `supervisor.restart{drain_timeout_ms}` (0 = cut now, for wedge bounces
85/// where a stuck request will never settle).
86pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89/// How long a drain waits after one busy-gauge probe completes before it asks
90/// again.
91///
92/// A drain re-probes on this short interval of its own rather than on the
93/// module's health cadence. The default cadence (30 s) equals the default
94/// drain ceiling, so a cadence-paced drain that found the module busy at its
95/// first probe never asked a second time: it waited out the whole ceiling and
96/// forced GOODBYE even after the gauges had read zero. The interval is counted
97/// from the moment the previous probe finished, and probes run one at a time,
98/// so a slow module is never asked more often than it can answer. Periodic
99/// health checks outside a drain keep using the configured cadence.
100const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101/// How long a restart waits for the exited process's output readers.
102///
103/// The restart does not depend on either output reader finishing. A stderr
104/// reader still running at this bound is left running, and its later output goes
105/// into the exited process's own section of the stderr ring (see
106/// `StderrRing::push_line_from`), ending naturally at EOF on its pipe. So the
107/// bound no longer decides whether a crash's last lines are kept: under load
108/// the reader may simply not have been scheduled yet, and cutting it there
109/// discarded exactly the lines that explained the crash.
110///
111/// What the bound still decides is when the tail starts reporting
112/// `Incomplete`: a pipe open past it usually means a descendant of the exited
113/// process still holds it, and the tail cannot claim to be whole until that
114/// pipe closes. That is also the only case in which the wait costs the restart
115/// anything, because a pipe with no other holder reaches EOF when the process
116/// exits. Under load a slow reader can show `Incomplete` briefly; it returns to
117/// `Captured` at EOF with nothing lost.
118///
119/// The stdout reader carries no ring, only the capture file. It also continues
120/// to EOF after this bound: child exit closes the writer but does not mean the
121/// reader has consumed all buffered bytes. Late output from either pipe can
122/// therefore reach the shared capture file after the successor starts.
123const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124/// Maximum number of supervised process spawn/exit facts retained per daemon incarnation.
125pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127/// Terminal Error code for a `supervisor.spawn_subscribe` stream the daemon
128/// dropped because the subscriber stopped draining its frames. The detail's
129/// `first_undelivered_cursor` names the first event it did not receive; the
130/// client resubscribes from the last cursor it did receive.
131pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135    reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136    deadline: tokio::time::Instant,
137    expected: Option<subc_os::FileIdentity>,
138    trampoline: Option<subc_os::FileIdentity>,
139    script: bool,
140    module_id: String,
141    #[cfg(test)]
142    confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147    // Cargo's unit-test executable lives in <profile>/deps; its fixture bin
148    // lives beside that directory. This honors custom CARGO_TARGET_DIR too.
149    let path = std::env::current_exe()
150        .unwrap()
151        .parent()
152        .unwrap()
153        .parent()
154        .unwrap()
155        .join("privacy-trampoline-fixture");
156    // Without the fixture every macOS spawn is refused, and the tests that
157    // spawn fail later as a module in state Failed, which names the wrong
158    // cause. `cargo test -p subc-daemon --lib` alone does not build it.
159    assert!(
160        path.exists(),
161        "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162         --bins --features test-support` or `cargo test -p subc-daemon` first",
163        path.display()
164    );
165    path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170    use std::io::Read;
171    let mut probe = std::process::Command::new(path)
172        .args(["__disclaim-exec", "--probe"])
173        .stdin(Stdio::null())
174        .stdout(Stdio::piped())
175        .stderr(Stdio::piped())
176        .spawn()
177        .map_err(|error| {
178            format!(
179                "privacy trampoline probe failed for {}: {error}",
180                path.display()
181            )
182        })?;
183    let deadline = std::time::Instant::now() + Duration::from_secs(5);
184    let status = loop {
185        match probe.try_wait() {
186            Ok(Some(status)) => break status,
187            Ok(None) if std::time::Instant::now() < deadline => {
188                std::thread::sleep(Duration::from_millis(5))
189            }
190            result => {
191                let _ = probe.kill();
192                let _ = probe.wait();
193                return Err(format!(
194                    "privacy trampoline probe failed or timed out for {}: {result:?}",
195                    path.display()
196                ));
197            }
198        }
199    };
200    let mut answer = String::new();
201    if let Some(stdout) = probe.stdout.take() {
202        let _ = stdout.take(256).read_to_string(&mut answer);
203    }
204    if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205        return Ok(());
206    }
207    let mut diagnostic = String::new();
208    if let Some(stderr) = probe.stderr.take() {
209        let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210    }
211    let cause = diagnostic
212        .trim()
213        .strip_prefix("ck-subc: own privacy identity refused: ")
214        .unwrap_or("binary does not implement the privacy trampoline protocol");
215    Err(format!(
216        "{cause}: probe of {} exited {status}",
217        path.display()
218    ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223    spec: &ModuleSpec,
224    roster: &ChildRoster,
225) -> Result<
226    (
227        Command,
228        Option<PrivacyExec>,
229        subc_os::privacy_identity::ExecAcknowledgement,
230    ),
231    SuperviseError,
232> {
233    let failure = |cause: String| {
234        warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235        SuperviseError::Spawn {
236            program: spec.program.clone(),
237            source: io::Error::other(cause),
238            cgroup_path: None,
239        }
240    };
241    let trampoline = roster.privacy_trampoline().map_err(failure)?;
242    // Resolve PATH with the same environment the Command will receive. For
243    // scripts retain the existing orphan-identity rule: the kernel chooses
244    // the interpreter, and its observed image is the one recorded. Do not
245    // duplicate the kernel's shebang/PATH interpreter resolution in Rust.
246    let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247        let path = spec
248            .env
249            .iter()
250            .find(|(key, _)| key == "PATH")
251            .map(|(_, value)| std::ffi::OsString::from(value))
252            .or_else(|| std::env::var_os("PATH"))
253            .unwrap_or_else(|| "/usr/bin:/bin".into());
254        std::env::split_paths(&path)
255            .map(|dir| dir.join(&spec.program))
256            .find(|path| path.is_file())
257            .unwrap_or_else(|| spec.program.clone())
258    } else {
259        spec.program.clone()
260    };
261    let expected = subc_os::file_identity(&program);
262    let trampoline_image = subc_os::file_identity(&trampoline);
263    let script = {
264        use std::io::Read;
265        let mut prefix = [0u8; 2];
266        std::fs::File::open(&program)
267            .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268    };
269    if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270        return Err(failure(
271            "privacy identity module executable is missing or is the trampoline itself".to_string(),
272        ));
273    }
274    let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275        .map_err(|error| failure(error.to_string()))?;
276    let reader =
277        tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278    let mut command = Command::new(&trampoline);
279    command
280        .arg("__disclaim-exec")
281        .arg(ack.fd().to_string())
282        .arg(&program);
283    ack.install(command.as_std_mut());
284    Ok((
285        command,
286        Some(PrivacyExec {
287            reader,
288            deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289            expected,
290            trampoline: trampoline_image,
291            script,
292            module_id: spec.module_id.clone(),
293            #[cfg(test)]
294            confirmation_pause: spec
295                .env
296                .iter()
297                .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298                .map(|(_, address)| (address.clone(), None)),
299        }),
300        ack,
301    ))
302}
303
304struct SupervisedChild {
305    child: Child,
306    exec: crate::activity::ExecOwner,
307    #[cfg(windows)]
308    nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309    #[cfg(target_os = "macos")]
310    privacy_exec: Option<PrivacyExec>,
311    /// Set once this launch's exec acknowledgement confirms the module image.
312    /// On macOS the pid first runs the `ck-subc` launch trampoline (see
313    /// `subc_os::privacy_identity`), which then replaces itself with the
314    /// module. The supervisor owns and can kill that pid from spawn, but
315    /// status readers report it only after this latch is set, so nothing
316    /// reports the trampoline's image as the module's.
317    #[cfg(target_os = "macos")]
318    report_ready: Arc<OnceLock<()>>,
319    /// Refusal before the module image was accepted, retained for terminal records.
320    spawn_failure: Option<String>,
321    /// The protocol this process was launched with. A reload can store a new
322    /// launch spec with a different protocol, but that takes effect only at the
323    /// next spawn, so this process keeps being handled by the protocol it
324    /// actually speaks.
325    protocol: ModuleProtocol,
326    /// This process's cgroup name: a bounded module/slot label followed by a
327    /// spawn suffix unique to this process (when cgroup placement is on). A
328    /// retired process in a slot may still be draining when a later one is
329    /// spawned into that slot, so the suffix keeps the later process out of
330    /// the retired one's cgroup, which is the domain a kill applies to.
331    #[cfg(target_os = "linux")]
332    module_id: String,
333    #[cfg(target_os = "linux")]
334    cgroup_placement: Option<subc_cgroup::Placement>,
335    /// The job that contains this child and every process it spawns (issue #109).
336    ///
337    /// Dropping this handle is what reaps a surviving tree when no supervisor
338    /// code runs — a daemon crash — because the job carries
339    /// `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`.
340    ///
341    /// That limit is not crash-only, and the difference is worth knowing: a
342    /// Windows daemon *stop* is `taskkill` or the scheduler's `/End` — the
343    /// SIGTERM handler is `#[cfg(unix)]` — so the daemon dies with no stop
344    /// notice and the kernel closes the job handle, `TerminateProcess`ing every
345    /// module at once. Before this change they survived that, saw EOF on the
346    /// control socket, and ran their own teardown; Unix keeps that path
347    /// deliberately, so a module can seal a WAL or close a capture rather than
348    /// be killed mid-write. So this trades graceful teardown on every Windows
349    /// daemon stop for containment on a crash, which is the right way round
350    /// today: orphaned GPU workers are a reported, recurring problem, and the
351    /// modules that write most heavily do not run on Windows.
352    ///
353    /// The fix is a real Windows stop path — the daemon draining before it
354    /// exits, the twin of the Unix SIGTERM handler. Once it exists, this limit
355    /// reaches only what the drain left behind, which is what it should reach.
356    #[cfg(windows)]
357    job: Option<subc_jobobject::JobObject>,
358    stdout_pump: Option<JoinHandle<()>>,
359    stderr_pump: Option<StderrPump>,
360    stderr_ring: Arc<Mutex<StderrRing>>,
361    /// Conservative monotonic bound sampled before `command.spawn()`, not when
362    /// identity recording finishes or the spawn generation is published.
363    spawned_at: Instant,
364    spawned_at_ms: u64,
365    spawned_from: PathBuf,
366    launch_spec_digest: LaunchSpecDigest,
367    spawned_file_identity: Option<SpawnedFileIdentity>,
368    process_start_time: Option<u64>,
369    process_identity: Option<ProcessIdentity>,
370    pid: u32,
371    /// This process's entry in the daemon's child roster, released when the
372    /// process is reaped or this handle is dropped.
373    roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377    fn id(&self) -> Option<u32> {
378        Some(self.pid)
379    }
380
381    fn process_identity(&self) -> Option<ProcessIdentity> {
382        self.process_identity
383    }
384
385    async fn wait(&mut self) -> io::Result<ExitStatus> {
386        #[cfg(target_os = "macos")]
387        self.confirm_privacy_exec().await;
388        // The roster entry is NOT released here. A daemon shutdown waits for the
389        // roster to empty and then exits the process, so releasing at the reap
390        // let it exit before the exit handler wrote this child's terminal record
391        // (the stderr drain and snapshot update sit in between), and the
392        // shutdown's own `daemon_shutdown` record was intermittently lost. The
393        // caller releases it after recording the exit (`release_roster`), and
394        // dropping the handle releases it too.
395        let result = self.child.wait().await;
396        #[cfg(target_os = "linux")]
397        if result.is_ok() {
398            if let Some(placement) = self.cgroup_placement.as_ref() {
399                cleanup_reaped_cgroup(placement, &self.module_id).await;
400                // Keep ownership while awaiting kernel population changes: a
401                // drain timeout may cancel this wait and then escalate/reap.
402                self.cgroup_placement = None;
403            }
404        }
405        result
406    }
407
408    #[cfg(target_os = "macos")]
409    async fn confirm_privacy_exec(&mut self) {
410        let Some(pending) = &mut self.privacy_exec else {
411            return;
412        };
413        let result = tokio::time::timeout_at(pending.deadline, async {
414            let mut record = Vec::new();
415            loop {
416                let mut ready = pending.reader.readable().await?;
417                let read = ready.try_io(|reader| {
418                    use std::io::Read;
419                    let mut reader = reader.get_ref();
420                    let mut buffer = [0u8; 256];
421                    reader.read(&mut buffer).map(|count| (count, buffer))
422                });
423                match read {
424                    Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425                    Ok(Ok((count, buffer))) => {
426                        if record.len() + count > 1024 {
427                            return Err(io::Error::other(
428                                "privacy exec refusal record is too long",
429                            ));
430                        }
431                        record.extend_from_slice(&buffer[..count]);
432                    }
433                    Ok(Err(error)) => return Err(error),
434                    Err(_) => continue,
435                }
436            }
437        })
438        .await;
439        #[cfg(test)]
440        let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441            use tokio::io::{AsyncReadExt, AsyncWriteExt};
442            if stream.is_none() {
443                *stream = Some(
444                    tokio::net::TcpStream::connect(address.as_str())
445                        .await
446                        .unwrap(),
447                );
448                stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449                stream
450                    .as_mut()
451                    .unwrap()
452                    .write_all(&self.pid.to_ne_bytes())
453                    .await
454                    .unwrap();
455            }
456            let mut reply = [0];
457            stream
458                .as_mut()
459                .unwrap()
460                .read_exact(&mut reply)
461                .await
462                .unwrap();
463            pending.confirmation_pause = None;
464            if &reply == b"F" {
465                Ok(Err(io::Error::other("injected late exec refusal")))
466            } else {
467                assert_eq!(&reply, b"X");
468                result
469            }
470        } else {
471            result
472        };
473        // Keep the reader in self across await: select cancellation must not
474        // discard the handshake or reset its original five-second deadline.
475        let pending = self.privacy_exec.as_ref().expect("pending exec");
476        let cause = match result {
477            Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478            Ok(Err(error)) => Some(format!(
479                "privacy identity exec acknowledgement failed: {error}"
480            )),
481            Ok(Ok(record)) if !record.is_empty() => Some(
482                std::str::from_utf8(&record)
483                    .ok()
484                    .and_then(|record| {
485                        record
486                            .trim()
487                            .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488                    })
489                    .filter(|cause| !cause.is_empty())
490                    .unwrap_or("invalid privacy exec refusal record")
491                    .to_string(),
492            ),
493            Ok(Ok(_)) => match self.child.try_wait() {
494                // Empty EOF is the exec acknowledgement. A real module may exit
495                // immediately, including with a reserved trampoline status; no
496                // image is admitted, and its ordinary exit contract stays intact.
497                Ok(Some(_status)) => None,
498                Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499                Ok(None) => {
500                    let image = observe_spawned_image(self.pid);
501                    if let Some(image) = image.filter(|image| {
502                        image.executable.is_some()
503                            && image.executable != pending.trampoline
504                            && (image.executable == pending.expected || pending.script)
505                    }) {
506                        if let Err(error) = self
507                            .roster_guard
508                            .as_ref()
509                            .map(|guard| guard.confirm_executable(image))
510                            .transpose()
511                        {
512                            Some(format!(
513                                "could not durably record confirmed module image: {error}"
514                            ))
515                        } else {
516                            let _ = self.report_ready.set(());
517                            info!(module_id = %pending.module_id, pid = self.pid,
518                                "module spawned with own privacy identity (responsibility disclaimed)");
519                            None
520                        }
521                    } else if image.is_none()
522                        || image.is_some_and(|image| image.executable.is_none())
523                    {
524                        // A process can exit between try_wait and the kernel
525                        // image read. Empty EOF already acknowledged exec, so
526                        // preserve that module's ordinary exit rather than
527                        // mislabel a disappearing image as trampoline refusal.
528                        // Keep pending in self across await so cancellation does
529                        // not discard validation or reset its original deadline.
530                        match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531                            Ok(Ok(_status)) => None,
532                            Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533                            Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534                        }
535                    } else {
536                        Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537                    }
538                }
539            },
540        };
541        let pending = self.privacy_exec.take().expect("pending exec");
542        // Exec confirmation describes launch success, not roster retirement.
543        // Keep even a reaped child recorded until its terminal history exists.
544        self.exec.0.finish(if cause.is_some() {
545            crate::activity::ExecState::Refused
546        } else {
547            crate::activity::ExecState::CommittedExec
548        });
549        if let Some(cause) = cause {
550            warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551            self.spawn_failure = Some(cause);
552            // No image is admitted on failure. Reach the entire fresh process
553            // group, including a module which spawned a helper before refusal.
554            if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555                let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556            }
557            let _ = self.child.start_kill();
558        }
559    }
560
561    /// Releases this child's daemon-shutdown roster entry once its exit has
562    /// been recorded. The pid is already reaped and free for reuse, so the
563    /// entry must not outlive the record any longer than that.
564    fn release_roster(&mut self) {
565        self.roster_guard = None;
566    }
567
568    /// Kill the child and, where containment is available, its process tree.
569    ///
570    /// `Child::kill` is `TerminateProcess` scoped to one pid, so a module with a
571    /// helper process leaked the helper — the Synapse embedding module's CUDA
572    /// worker holds the GPU allocation, so the leak cost VRAM until the next
573    /// restart of something else. Terminating the job reaches grandchildren that
574    /// a tree walk cannot, including one whose parent has already exited and
575    /// been reparented away.
576    ///
577    /// Best-effort like `request_graceful_stop`: a job failure is logged and the
578    /// direct-child kill still decides the outcome, so containment can never
579    /// change whether a module is reported as stopped.
580    fn start_kill(&mut self) -> io::Result<()> {
581        #[cfg(windows)]
582        if let Some(job) = &self.job {
583            if let Err(error) = job.terminate() {
584                debug!(
585                    error = %error,
586                    "job termination failed; the direct-child kill still owns the outcome"
587                );
588            }
589        }
590        #[cfg(target_os = "linux")]
591        kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592        self.child.start_kill()
593    }
594
595    async fn drain_stderr(&mut self, module_id: &str) {
596        if let Some(pump) = self.stdout_pump.take() {
597            settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598        }
599
600        let Some(pump) = self.stderr_pump.take() else {
601            return;
602        };
603        settle_stderr_pump(
604            module_id,
605            &self.stderr_ring,
606            pump,
607            STDERR_PUMP_DRAIN_TIMEOUT,
608        )
609        .await;
610    }
611}
612
613/// Bound the restart's wait, not the lifetime of the exited child's stdout
614/// reader. Slow pipe reads or capture writes must not turn the bound into lost
615/// output; dropping the handle lets the reader finish at EOF, like stderr.
616async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617    match timeout(bound, &mut pump).await {
618        Ok(Ok(())) => {}
619        Ok(Err(error)) => {
620            warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621        }
622        Err(_) => {
623            drop(pump);
624            warn!(
625                module_id,
626                waited = ?bound,
627                "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628            );
629        }
630    }
631}
632
633/// The reader task for one process's stderr, with the ring generation its
634/// lines are attributed to.
635struct StderrPump {
636    task: JoinHandle<()>,
637    generation: u64,
638}
639
640/// Retire an exited process's stderr reader and wait up to `bound` for it to
641/// reach EOF. A reader still running at the bound is detached, not stopped: it
642/// keeps filling the exited process's section of the ring until its pipe
643/// closes, and the tail reads `Incomplete` until then. See
644/// [`STDERR_PUMP_DRAIN_TIMEOUT`] for why.
645async fn settle_stderr_pump(
646    module_id: &str,
647    ring: &Arc<Mutex<StderrRing>>,
648    pump: StderrPump,
649    bound: Duration,
650) {
651    let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652    let StderrPump {
653        mut task,
654        generation,
655    } = pump;
656    lock().retire_pump(generation);
657    match timeout(bound, &mut task).await {
658        Ok(Ok(())) => {}
659        Ok(Err(err)) => {
660            let mut ring = lock();
661            ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662            ring.finish_pump(generation);
663            warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664        }
665        Err(_) => {
666            // Dropping the handle detaches the task; it ends at EOF on its pipe.
667            drop(task);
668            lock().mark_pump_late(
669                generation,
670                format!(
671                    "stderr of the exited process had not reached EOF {bound:?} after it was \
672                     retired (a descendant may still hold the pipe open); lines it still \
673                     writes are kept in that process's section"
674                ),
675            );
676            warn!(
677                module_id,
678                waited = ?bound,
679                "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680            );
681        }
682    }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686    static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687    EVENTS.get_or_init(|| {
688        let (sender, _receiver) = watch::channel(0);
689        sender
690    })
691}
692
693pub(crate) fn notify_registration_release() {
694    let events = registration_release_events();
695    let next_generation = (*events.borrow()).wrapping_add(1);
696    events.send_replace(next_generation);
697}
698
699/// How to launch one singleton module process.
700#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702    pub module_id: String,
703    pub program: PathBuf,
704    pub args: Vec<String>,
705    pub env: Vec<(String, String)>,
706    /// When true this is a reserved module: each spawn gets a fresh one-time launch
707    /// nonce that the child must echo in its HELLO, so only the daemon-spawned
708    /// process can register this module_id (a security-boundary module like the
709    /// credential vault must not be impersonable while it is down/restarting).
710    pub reserved: bool,
711    /// Module-id prefixes this supervised module owns for reserved HELLO checks.
712    /// Prefixes come from daemon config and must end in `:` before they reach the
713    /// supervisor; the owner module's current spawn nonce authorizes claims under
714    /// each prefix.
715    pub reserved_prefixes: Vec<String>,
716    /// The wire protocol this module speaks, as DECLARED in daemon config.
717    ///
718    /// [`ModuleProtocol::None`] changes five things and nothing else: wire health
719    /// probing is suppressed (an optional HTTP probe can replace it), teardown sends SIGTERM before waiting,
720    /// `route.open` is refused, the spawn passes NO `--subc <path>` argument
721    /// and NO launch nonce, and a clean exit the daemon did not request is
722    /// restarted as a crash rather than recorded as a stop (see `on_child_exit`:
723    /// a stock program exits 0 on a stray SIGTERM, and a stop would leave it
724    /// down for good). `SUBC_MODULE_ID` still goes into the environment,
725    /// because a process ignores an environment variable it does not read.
726    ///
727    /// The argument is the part that cannot be "harmless to a process that
728    /// ignores it": a stock binary exits on an unknown flag before it listens
729    /// (`nats-server`: "flag provided but not defined: -subc"), which is how the
730    /// first conformance run against this mode found it. The nonce is withheld
731    /// because a process that will never present it gains nothing from holding
732    /// it, and a secret in the environment of a process that does not need it is
733    /// a leak surface for no benefit.
734    pub protocol: ModuleProtocol,
735    /// Whether two processes of this module may run at once, which is what a
736    /// blue/green swap does for the length of its overlap. Declared in daemon
737    /// config because the daemon must be able to answer it while the module is
738    /// down, and so a module cannot talk itself into it after registering.
739    pub overlap: ModuleOverlap,
740}
741
742/// Only the configured launch inputs enter this value; spawned connection paths,
743/// nonces and inherited environment never do. Its bytes remain daemon-private.
744#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749        f.write_str("LaunchSpecDigest([redacted])")
750    }
751}
752
753impl LaunchSpecDigest {
754    pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755        fn field(hash: &mut Sha256, bytes: &[u8]) {
756            hash.update((bytes.len() as u64).to_le_bytes());
757            hash.update(bytes);
758        }
759        let mut hash = Sha256::new();
760        if let Some(program) = spec.program.to_str() {
761            field(&mut hash, b"utf8-program");
762            field(&mut hash, program.as_bytes());
763        } else {
764            // The daemon config is UTF-8, but in-process callers may provide
765            // native paths: lossy conversion would collapse distinct programs.
766            #[cfg(unix)]
767            {
768                use std::os::unix::ffi::OsStrExt;
769                field(&mut hash, b"unix-program");
770                field(&mut hash, spec.program.as_os_str().as_bytes());
771            }
772            #[cfg(windows)]
773            {
774                use std::os::windows::ffi::OsStrExt;
775                field(&mut hash, b"windows-program");
776                let bytes = spec
777                    .program
778                    .as_os_str()
779                    .encode_wide()
780                    .flat_map(u16::to_le_bytes)
781                    .collect::<Vec<_>>();
782                field(&mut hash, &bytes);
783            }
784            #[cfg(not(any(unix, windows)))]
785            field(&mut hash, spec.program.to_string_lossy().as_bytes());
786        }
787        hash.update((spec.args.len() as u64).to_le_bytes());
788        for arg in &spec.args {
789            field(&mut hash, arg.as_bytes());
790        }
791        let mut env = spec.env.iter().collect::<Vec<_>>();
792        env.sort_unstable();
793        hash.update((env.len() as u64).to_le_bytes());
794        for (key, value) in env {
795            field(&mut hash, key.as_bytes());
796            field(&mut hash, value.as_bytes());
797        }
798        field(
799            &mut hash,
800            match spec.protocol {
801                ModuleProtocol::Subc => b"subc",
802                ModuleProtocol::None => b"none",
803            },
804        );
805        Self(hash.finalize().into())
806    }
807}
808
809/// Whether a module tolerates a second process of itself running alongside.
810///
811/// Most modules are single-writer on their store (a WAL, a capture log, a
812/// resident index behind a writer barrier), and two processes on one store
813/// corrupt it. So a swap, which overlaps the old and new process by design,
814/// is refused unless the module's config opts in with `overlap: "safe"`.
815#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817    /// Never run two processes of this module at once. The default.
818    #[default]
819    Exclusive,
820    /// The module has said a second process of itself is harmless for the
821    /// length of a swap.
822    ///
823    /// Declare it only if a second instance can run for a few seconds without
824    /// touching ANY single-writer store: every database, WAL, index, projector
825    /// and scheduled job the module owns. A lease on part of that state is not
826    /// enough. broca's session lease guards WAL appends while its run index, its
827    /// store projector and its archive fold timer (which unlinks live WAL files)
828    /// stay single-writer, so broca is exclusive despite holding a lease. The
829    /// refusal only fires after this has been decided, so the decision is the
830    /// check.
831    Safe,
832}
833
834impl ModuleOverlap {
835    pub fn as_str(self) -> &'static str {
836        match self {
837            Self::Exclusive => "exclusive",
838            Self::Safe => "safe",
839        }
840    }
841}
842
843/// Environment variable telling a spawned module which case it was started
844/// for, before it sends HELLO. Only a swap candidate carries it, as
845/// [`SPAWN_ROLE_SWAP_CANDIDATE`]; every other spawn has it removed.
846///
847/// It chooses a warm-up budget, nothing else: a swap candidate can warm for
848/// longer because nobody waits on it, while a plain restart must flip ready
849/// quickly because callers see `module_warming` until it does. Absence means
850/// plain restart, the safe reading. The daemon trusts nothing about it; the
851/// candidate is proven by its launch nonce at HELLO.
852pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853/// The one value of [`SUBC_SPAWN_ROLE_ENV`] the daemon sets.
854pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855/// How long a swap waits for its candidate to register and declare itself
856/// ready when the operator does not say. A module warming as a swap candidate
857/// may take up to 90 s (aft's ceiling, the largest in the fleet), so the
858/// daemon allows that plus time to start the process and send HELLO.
859pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861/// Bounded restart policy for crash exits.
862///
863/// `max_restarts` is the number of replacement processes allowed after the
864/// initial spawn WITHIN `window`. After that many crash restarts inside one
865/// window the module enters [`ModuleState::Failed`] and the supervisor stops
866/// the crash loop.
867///
868/// The budget is a RATE, not a lifetime total. It used to be a lifetime total,
869/// and that only survived because crashes were rare: a module that crashed
870/// three times across a week was disabled forever by crashes that had nothing
871/// to do with each other. That stopped being survivable once modules began
872/// exiting non-zero whenever the daemon's connection to them drops, because
873/// then every daemon-side connection drop spends a unit of the same budget and
874/// one flappy hour permanently stops a healthy module. Restarts older than
875/// `window` release their slot, so a module that crashed twice yesterday has a
876/// full budget today, while a genuine crash loop -- which is fast by
877/// definition -- still reaches the cap and stops.
878#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880    pub max_restarts: u32,
881    /// Base delay before a crash replacement. The actual delay escalates with
882    /// the number of recent crash replacements and is capped by `max_backoff`.
883    pub backoff: Duration,
884    /// Maximum delay before a crash replacement.
885    pub max_backoff: Duration,
886    /// The span `max_restarts` is counted over. `Duration::ZERO` makes the
887    /// budget effectively infinite (nothing is ever in-window), which is why
888    /// daemon config refuses `window_secs: 0` rather than quietly accepting it.
889    pub window: Duration,
890}
891
892impl RestartPolicy {
893    /// A policy with the default crash window. Callers that care about the
894    /// window say so with [`Self::with_window`]; the ones that do not are
895    /// asking for the standard rate limit, not for no limit.
896    pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897        Self {
898            max_restarts,
899            backoff,
900            max_backoff: DEFAULT_MAX_BACKOFF,
901            window: DEFAULT_RESTART_WINDOW,
902        }
903    }
904
905    pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906        self.max_backoff = max_backoff;
907        self
908    }
909
910    pub fn with_window(mut self, window: Duration) -> Self {
911        self.window = window;
912        self
913    }
914
915    /// Calculate the capped exponential delay for the next crash replacement.
916    /// `restart_in_window` is zero for the first replacement after an operator
917    /// action (restart, reload, re-enable) cleared the crash ring, or after all
918    /// older crash replacements have aged out of the window.
919    fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920        if self.backoff.is_zero() || self.max_backoff.is_zero() {
921            return Duration::ZERO;
922        }
923
924        let mut delay = self.backoff;
925        for _ in 0..restart_in_window {
926            if delay >= self.max_backoff {
927                return self.max_backoff;
928            }
929            delay = delay
930                .checked_mul(10)
931                .unwrap_or(self.max_backoff)
932                .min(self.max_backoff);
933        }
934        delay.min(self.max_backoff)
935    }
936
937    /// The one sentence that explains a budget-exhausted stop, used for both the
938    /// log line and the terminal record so the two cannot drift. It names the
939    /// window because `max_restarts=3` alone reads as a lifetime cap, which is
940    /// exactly what this budget is not.
941    fn budget_exhausted_detail(&self) -> String {
942        format!(
943            "crash budget exhausted: max_restarts={} within window_secs={}",
944            self.max_restarts,
945            self.window.as_secs()
946        )
947    }
948}
949
950impl Default for RestartPolicy {
951    fn default() -> Self {
952        Self {
953            max_restarts: DEFAULT_MAX_RESTARTS,
954            backoff: DEFAULT_BACKOFF,
955            max_backoff: DEFAULT_MAX_BACKOFF,
956            window: DEFAULT_RESTART_WINDOW,
957        }
958    }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963    restart_in_window: u32,
964    delay: Duration,
965}
966
967/// Whether the daemon itself will bring this module back after the exit being
968/// handled: it is enabled AND its in-window crash restarts are below the cap.
969///
970/// Takes `&mut` because reading the budget prunes it. Instants that fell out of
971/// the window are dropped here rather than by a timer, so the count is right
972/// the moment somebody asks and no bookkeeping runs for idle modules.
973fn daemon_will_restart(
974    state: &mut SupervisorSnapshot,
975    policy: &RestartPolicy,
976    now: Instant,
977) -> bool {
978    state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988    Report,
989    Restart,
990    Alert,
991}
992
993impl fmt::Display for HealthAction {
994    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995        f.write_str(match self {
996            Self::Report => "report",
997            Self::Restart => "restart",
998            Self::Alert => "alert",
999        })
1000    }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005    /// Optional loopback HTTP endpoint for a managed non-wire process.
1006    /// Changing it applies live on rescan; the process protocol changes only
1007    /// at its next spawn.
1008    pub http: Option<String>,
1009    pub cadence: Duration,
1010    pub deadline: Duration,
1011    pub failure_threshold: u32,
1012    pub on_degraded: HealthAction,
1013    pub on_failing: HealthAction,
1014    pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018    fn default() -> Self {
1019        Self {
1020            http: None,
1021            cadence: DEFAULT_HEALTH_CADENCE,
1022            deadline: DEFAULT_HEALTH_DEADLINE,
1023            failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024            on_degraded: HealthAction::Report,
1025            on_failing: HealthAction::Report,
1026            critical: false,
1027        }
1028    }
1029}
1030
1031/// The supervisor's view of one module's health, relayed to clients over
1032/// channel-0 and rendered by `ck health`.
1033///
1034/// THIS TYPE IS WHERE THE ABSENCE MEANINGS ARE CREATED, which is why they are
1035/// stated here rather than only at the wire type a consumer reads. A reader can
1036/// look up what `None` means; only a writer can silently change it, and the
1037/// writer has no reason to go looking at a downstream contract before editing.
1038///
1039/// `last_probe_ms: None` MEANS NEVER PROBED, not probed-long-ago. It is cleared
1040/// back to `None` on re-registration precisely so a respawned module does not
1041/// carry its predecessor's timestamp — so an old value and an absent one call for
1042/// opposite readings, and anything that defaulted this to a number would make a
1043/// never-probed module indistinguishable from one probed at the epoch.
1044///
1045/// `detail` and `metrics` are `None` when the module published none on this
1046/// probe, which does not mean it reported nothing wrong — it is also the shape
1047/// when the probe never reached it. `last_probe_ms` is what separates those.
1048#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050    pub status: SupervisorHealthStatus,
1051    pub last_probe_ms: Option<u64>,
1052    pub detail: Option<String>,
1053    pub metrics: Option<Value>,
1054    pub consecutive_failures: u32,
1055    /// Number of replies received after a recurring health probe's deadline.
1056    /// Unlike a timeout, every increment proves the module was alive.
1057    pub late_answer_count: u64,
1058    /// End-to-end latency of the newest late reply, measured from probe start.
1059    pub last_late_answer_latency_ms: Option<u64>,
1060    pub last_action: Option<String>,
1061    /// Set together with `last_action`; the pair moves as one, and both being
1062    /// absent means no escalation has ever been taken rather than that the last
1063    /// one succeeded.
1064    pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068    fn default() -> Self {
1069        Self {
1070            status: SupervisorHealthStatus::Unknown,
1071            last_probe_ms: None,
1072            detail: None,
1073            metrics: None,
1074            consecutive_failures: 0,
1075            late_answer_count: 0,
1076            last_late_answer_latency_ms: None,
1077            last_action: None,
1078            last_action_ms: None,
1079        }
1080    }
1081}
1082
1083/// Typed lifecycle state for a supervised module.
1084#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086    Starting,
1087    Running,
1088    Unresponsive,
1089    Restarting,
1090    Draining,
1091    Stopped,
1092    Failed,
1093    Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098        f.write_str(match self {
1099            Self::Starting => "starting",
1100            Self::Running => "running",
1101            Self::Unresponsive => "unresponsive",
1102            Self::Restarting => "restarting",
1103            Self::Draining => "draining",
1104            Self::Stopped => "stopped",
1105            Self::Failed => "failed",
1106            Self::Disabled => "disabled",
1107        })
1108    }
1109}
1110
1111/// Supervisor classification of a child-process exit.
1112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114    Clean,
1115    Crash,
1116    DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120    fn from(kind: ExitKind) -> Self {
1121        match kind {
1122            ExitKind::Clean => Self::Clean,
1123            ExitKind::Crash => Self::Crash,
1124            ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125        }
1126    }
1127}
1128
1129/// Exact process identity retained when a supervised module registers its
1130/// connection. PID reuse makes a PID alone insufficient evidence of ownership.
1131#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133    pub(crate) pid: u32,
1134    pub(crate) start_time: u64,
1135}
1136
1137/// Last observed child exit, if any.
1138#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140    pub kind: ExitKind,
1141    pub code: Option<i32>,
1142    pub signal: Option<i32>,
1143    pub at_ms: u64,
1144}
1145
1146/// Point-in-time module status answerable by subc without forwarding to the
1147/// module process.
1148#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150    pub module_id: String,
1151    /// A previous daemon's unresolved process, not a child owned by this supervisor.
1152    pub waiting_for_previous_pid: Option<u32>,
1153    /// Why the previous instance still prevents a new start.
1154    pub waiting_for_previous_detail: Option<String>,
1155    /// The daemon's Windows per-spawn record: `pipe` after the process it
1156    /// spawned read all nonce bytes, otherwise `env` for the offered copy.
1157    /// List/status reports use this record, not the source the module sends
1158    /// in its registration message's manifest.provenance.launch_nonce_source.
1159    pub launch_nonce_source: Option<String>,
1160    pub state: ModuleState,
1161    pub enabled: bool,
1162    pub process_alive: bool,
1163    pub registration_active: bool,
1164    /// The module's declared wire protocol, carried beside `live` because it is
1165    /// what makes `live` readable: the two fields answer one question together.
1166    /// While a process is alive this is its launch declaration, not a later
1167    /// pending-reload edit. When down it is the configured next launch protocol.
1168    pub protocol: ModuleProtocol,
1169    /// Whether the module is serving, under the strongest definition the daemon
1170    /// can assert for its protocol.
1171    ///
1172    /// A subc module must also be REGISTERED: its process being alive says
1173    /// nothing about whether it can take a request. A `protocol: "none"` module
1174    /// never registers, so that term is dropped and this falls back to "enabled,
1175    /// running, and the process the daemon launched is alive" -- which is all
1176    /// the daemon observes about a process that speaks no subc wire. It stays a
1177    /// `bool` on the wire for compatibility; renderers pair it with `protocol`
1178    /// rather than printing it bare.
1179    pub live: bool,
1180    /// Crash restarts spent INSIDE `restart_window` as of this read. Older
1181    /// restarts have already released their slot, so this count can go down
1182    /// without anybody touching the module.
1183    pub restart_count: u32,
1184    /// Replacement processes spawned over this module's entire supervisor lifetime;
1185    /// unlike `restart_count`, this value is never reset by an operator action
1186    /// and never falls out of a window.
1187    pub lifetime_restarts: u32,
1188    pub spawn_generation: u64,
1189    /// The budget `restart_count` is spent against. Carried alongside the count
1190    /// because the count alone does not say how close the module is to being
1191    /// disabled, and reporting one without the other is what makes an
1192    /// about-to-be-retired module look ordinary.
1193    pub max_restarts: u32,
1194    /// The span `restart_count` is counted over. Carried with the pair above for
1195    /// the same reason they are carried together: "2 of 3" means one thing for a
1196    /// ten-minute window and something else entirely for a lifetime.
1197    pub restart_window: Duration,
1198    /// Effective drain and restart timing policy used by this running module.
1199    /// These values are carried together with the restart budget so status
1200    /// readers can compare configured intent with what the supervisor applied.
1201    pub drain_timeout: Duration,
1202    pub restart_backoff: Duration,
1203    pub restart_max_backoff: Duration,
1204    /// The module's process. On macOS this stays absent while the `ck-subc`
1205    /// launch trampoline is still running in that pid, and appears once the
1206    /// exec acknowledgement confirms the module image has replaced it. Launch
1207    /// time and the supervisor's own hold on the process are unaffected.
1208    pub pid: Option<u32>,
1209    pub spawned_at_ms: Option<u64>,
1210    pub spawned_from: Option<PathBuf>,
1211    pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212    pub process_start_time: Option<u64>,
1213    pub last_exit: Option<ExitReport>,
1214    pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220    generation: u64,
1221    turn: u64,
1222    registered_connection: Option<ConnectionId>,
1223    next_probe_at: Option<Instant>,
1224    wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229    exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230    #[cfg(windows)]
1231    pipe_consumed: Option<Arc<OnceLock<()>>>,
1232    /// Counts running-child loop turns, not executor polls of a parked wait.
1233    #[cfg(test)]
1234    actor_turns: u64,
1235    /// Running select after exec confirmation and with no unconsumed registry
1236    /// event. Recording a spawn or starting a loop turn is not this barrier.
1237    #[cfg(test)]
1238    actor_select: Option<ActorSelectCheckpoint>,
1239    state: ModuleState,
1240    enabled: bool,
1241    process_alive: bool,
1242    spawned_protocol: Option<ModuleProtocol>,
1243    spawn_failure: Option<String>,
1244    waiting_for_previous_pid: Option<u32>,
1245    waiting_for_previous_detail: Option<String>,
1246    /// When each crash restart was spent, oldest first. This IS the crash
1247    /// budget: its in-window length is the count an operator sees and the count
1248    /// the restart decision is made against, so there is no second counter that
1249    /// can disagree with it. Bounded by `max_restarts`, and cleared by the same
1250    /// operator actions that used to zero the old lifetime counter.
1251    crash_restarts: VecDeque<Instant>,
1252    lifetime_restarts: u32,
1253    /// Successful child spawns in this daemon incarnation.
1254    ///
1255    /// `lifetime_restarts` was considered and rejected: it starts at zero
1256    /// (line 640), successful initial/operator spawns in `set_running` do not
1257    /// increment it (lines 5264-5274), and crash/deliberate retry bookkeeping
1258    /// increments before a successful replacement exists (lines 604, 3846,
1259    /// and 3921), so a failed spawn can consume it. This counter moves only
1260    /// when a live PID is accepted below.
1261    spawn_generation: u64,
1262    pid: Option<u32>,
1263    #[cfg(target_os = "macos")]
1264    report_ready: Option<Arc<OnceLock<()>>>,
1265    /// Last reaped child, retained after current process facts are cleared.
1266    reaped_pid: Option<u32>,
1267    /// Whether a replacement has already been scheduled for the current process.
1268    respawn_pending: bool,
1269    /// A second restart is waiting for the replacement already scheduled.
1270    coalesced_restart_pending: bool,
1271    spawned_at_ms: Option<u64>,
1272    spawned_from: Option<PathBuf>,
1273    launch_spec_digest: Option<LaunchSpecDigest>,
1274    spawned_file_identity: Option<SpawnedFileIdentity>,
1275    process_start_time: Option<u64>,
1276    deliberate_severance: Option<ProcessIdentity>,
1277    last_exit: Option<ExitReport>,
1278    /// Diagnostic attached to the next drain's terminal record, if any.
1279    drain_disposition_detail: Option<String>,
1280    health: ModuleHealthStatus,
1281    /// Whether the current process was started as a swap candidate and so
1282    /// lives in the module's alternate cgroup. The next swap's candidate takes
1283    /// the other one, so the two processes of a swap never share a cgroup. A
1284    /// plain spawn always uses the primary cgroup.
1285    in_alternate_slot: bool,
1286    /// Whether the current `Draining` state ends in a replacement process
1287    /// (restart, reload, health restart) rather than a stop. Only meaningful
1288    /// while `state` is `Draining`; every entry into that state rewrites it.
1289    /// It is what lets route.open answer the retryable `module_reloading` to a
1290    /// consumer that reaches a still-registered process mid-restart, instead of
1291    /// the `supervisor_not_live` a stop or disable deserves.
1292    draining_to_replace: bool,
1293    /// Whether a configuration update has been applied since the current
1294    /// process was spawned, so that process runs an older spec than the one
1295    /// the supervisor now holds. A queued restart is only coalesced into a
1296    /// fresher process when this is false: a restart requested to pick up a
1297    /// new configuration must not be satisfied by a process that predates it.
1298    configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302    /// The pid that status, provenance and resource readings may report. While
1303    /// the launch trampoline still runs in the pid, reading its executable or
1304    /// resource use would describe `ck-subc`, not the module, so none is
1305    /// reported until the exec acknowledgement confirms the module image.
1306    fn reported_pid(&self) -> Option<u32> {
1307        #[cfg(target_os = "macos")]
1308        if self
1309            .report_ready
1310            .as_ref()
1311            .is_some_and(|ready| ready.get().is_none())
1312        {
1313            return None;
1314        }
1315        self.pid
1316    }
1317
1318    fn starting() -> Self {
1319        Self::new(ModuleState::Starting, true)
1320    }
1321
1322    fn disabled() -> Self {
1323        Self::new(ModuleState::Disabled, false)
1324    }
1325
1326    /// Crash restarts still inside `window`, having dropped the ones that are
1327    /// not. Pruning on read is what makes the budget a rate: an instant older
1328    /// than the window stops holding a slot the moment anybody counts.
1329    fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330        while let Some(oldest) = self.crash_restarts.front() {
1331            if now.duration_since(*oldest) > window {
1332                self.crash_restarts.pop_front();
1333            } else {
1334                break;
1335            }
1336        }
1337        u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338    }
1339
1340    /// Spend one unit of the crash budget and record the restart in the ledger.
1341    ///
1342    /// The ring is bounded by the cap because more than `max_restarts` in-window
1343    /// instants can never be reached (the caller refuses the restart first), so
1344    /// anything beyond that is an unbounded queue waiting to happen.
1345    fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346        self.crash_restarts.push_back(now);
1347        while self.crash_restarts.len() > policy.max_restarts as usize {
1348            self.crash_restarts.pop_front();
1349        }
1350        self.lifetime_restarts += 1;
1351    }
1352
1353    /// Reserve one crash-restart slot and calculate the delay before respawning.
1354    /// The count is captured before recording this restart, so the first retry
1355    /// uses the base delay and each later in-window retry escalates once.
1356    fn next_crash_restart(
1357        &mut self,
1358        policy: &RestartPolicy,
1359        now: Instant,
1360    ) -> Option<CrashRestartSchedule> {
1361        let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362        if restart_in_window >= policy.max_restarts {
1363            return None;
1364        }
1365        self.record_crash_restart(policy, now);
1366        Some(CrashRestartSchedule {
1367            restart_in_window,
1368            delay: policy.delay_for_restart(restart_in_window),
1369        })
1370    }
1371
1372    /// Give the module its full budget back, as an operator restart, reload, or
1373    /// re-enable does. `lifetime_restarts` deliberately does not move: it is the
1374    /// ledger of what actually happened, and an operator action does not unmake
1375    /// the crashes.
1376    fn clear_crash_restarts(&mut self) {
1377        self.crash_restarts.clear();
1378    }
1379
1380    fn new(state: ModuleState, enabled: bool) -> Self {
1381        Self {
1382            exec_outcome: None,
1383            #[cfg(test)]
1384            actor_turns: 0,
1385            #[cfg(test)]
1386            actor_select: None,
1387            state,
1388            enabled,
1389            process_alive: false,
1390            #[cfg(windows)]
1391            pipe_consumed: None,
1392            spawned_protocol: None,
1393            spawn_failure: None,
1394            waiting_for_previous_pid: None,
1395            waiting_for_previous_detail: None,
1396            crash_restarts: VecDeque::new(),
1397            lifetime_restarts: 0,
1398            spawn_generation: 0,
1399            pid: None,
1400            #[cfg(target_os = "macos")]
1401            report_ready: None,
1402            reaped_pid: None,
1403            respawn_pending: false,
1404            coalesced_restart_pending: false,
1405            spawned_at_ms: None,
1406            spawned_from: None,
1407            launch_spec_digest: None,
1408            spawned_file_identity: None,
1409            process_start_time: None,
1410            deliberate_severance: None,
1411            last_exit: None,
1412            drain_disposition_detail: None,
1413            health: ModuleHealthStatus::default(),
1414            in_alternate_slot: false,
1415            draining_to_replace: false,
1416            configuration_updated_since_spawn: false,
1417        }
1418    }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427    version: u8,
1428    frames: mpsc::Sender<Frame>,
1429    /// Tells this subscriber's forwarder that it was dropped for lagging, and
1430    /// from which event. The full frame channel cannot carry that news, so it
1431    /// travels beside it; see `SpawnEventFeed::subscribe`.
1432    lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437    daemon_incarnation: String,
1438    seq: u64,
1439    capacity: usize,
1440    live: HashMap<String, LiveSpawn>,
1441    generations: HashMap<String, u64>,
1442    events: VecDeque<SpawnEvent>,
1443    subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447    fn default() -> Self {
1448        Self {
1449            daemon_incarnation: "unconfigured".to_string(),
1450            seq: 0,
1451            capacity: SPAWN_EVENT_RING_CAPACITY,
1452            live: HashMap::new(),
1453            generations: HashMap::new(),
1454            events: VecDeque::new(),
1455            subscribers: HashMap::new(),
1456        }
1457    }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462    Arc<Mutex<SpawnEventState>>,
1463    #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468    ForeignIncarnation { current: String },
1469    TooOld { oldest: SpawnCursor },
1470    Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474    fn configure_incarnation(&self, daemon_incarnation: String) {
1475        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476        state.daemon_incarnation = daemon_incarnation;
1477        state.seq = 0;
1478        state.live.clear();
1479        state.generations.clear();
1480        state.events.clear();
1481        state.subscribers.clear();
1482    }
1483
1484    fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485        SpawnCursor {
1486            daemon_incarnation: state.daemon_incarnation.clone(),
1487            seq: state.seq,
1488        }
1489    }
1490
1491    fn snapshot(&self) -> SpawnSnapshot {
1492        let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493        let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494        live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495        SpawnSnapshot {
1496            cursor: Self::cursor(&state),
1497            ring_bound: state.capacity as u64,
1498            live,
1499        }
1500    }
1501
1502    fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504        let generation = state
1505            .generations
1506            .get(module_id)
1507            .copied()
1508            .unwrap_or(0)
1509            .checked_add(1)
1510            .expect("spawn generation exhausted");
1511        state.generations.insert(module_id.to_string(), generation);
1512        let live = LiveSpawn {
1513            module_id: module_id.to_string(),
1514            spawn_generation: generation,
1515            pid,
1516            spawned_at_ms,
1517        };
1518        state.live.insert(module_id.to_string(), live);
1519        Self::emit_locked(
1520            &mut state,
1521            SpawnEventKind::Spawned,
1522            module_id.to_string(),
1523            generation,
1524            pid,
1525            None,
1526            None,
1527        );
1528        generation
1529    }
1530
1531    fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533        let Some(live) = state.live.remove(module_id) else {
1534            warn!(
1535                module_id,
1536                "terminal record had no live spawn event identity"
1537            );
1538            return;
1539        };
1540        Self::emit_locked(
1541            &mut state,
1542            SpawnEventKind::Exited,
1543            module_id.to_string(),
1544            live.spawn_generation,
1545            live.pid,
1546            exit_code,
1547            exit_signal,
1548        );
1549    }
1550
1551    /// Report the exit of a process that a swap has already replaced.
1552    ///
1553    /// `emit_exited` removes the module's live entry, which after a swap's
1554    /// cutover describes the promoted candidate, not the old process now
1555    /// exiting. This emits the old generation's exit and leaves the live entry
1556    /// alone unless it still names that generation.
1557    fn emit_superseded_exited(
1558        &self,
1559        module_id: &str,
1560        spawn_generation: u64,
1561        pid: u32,
1562        exit_code: Option<i32>,
1563        exit_signal: Option<i32>,
1564    ) {
1565        let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566        if state
1567            .live
1568            .get(module_id)
1569            .is_some_and(|live| live.spawn_generation == spawn_generation)
1570        {
1571            state.live.remove(module_id);
1572        }
1573        Self::emit_locked(
1574            &mut state,
1575            SpawnEventKind::Exited,
1576            module_id.to_string(),
1577            spawn_generation,
1578            pid,
1579            exit_code,
1580            exit_signal,
1581        );
1582    }
1583
1584    #[allow(clippy::too_many_arguments)]
1585    fn emit_locked(
1586        state: &mut SpawnEventState,
1587        kind: SpawnEventKind,
1588        module_id: String,
1589        spawn_generation: u64,
1590        pid: u32,
1591        exit_code: Option<i32>,
1592        exit_signal: Option<i32>,
1593    ) {
1594        state.seq = state
1595            .seq
1596            .checked_add(1)
1597            .expect("spawn event sequence exhausted");
1598        let event = SpawnEvent {
1599            cursor: Self::cursor(state),
1600            kind,
1601            module_id,
1602            spawn_generation,
1603            pid,
1604            exit_code,
1605            exit_signal,
1606        };
1607        state.events.push_back(event.clone());
1608        while state.events.len() > state.capacity {
1609            state.events.pop_front();
1610        }
1611        let body = match serde_json::to_vec(&event) {
1612            Ok(body) => body,
1613            Err(error) => {
1614                error!(%error, "failed to serialize supervisor spawn event");
1615                return;
1616            }
1617        };
1618        state.subscribers.retain(|(connection_id, corr), subscriber| {
1619            let frame = Frame::build_with_version(
1620                subscriber.version,
1621                FrameType::StreamData,
1622                control_flags(),
1623                0,
1624                0,
1625                *corr,
1626                body.clone(),
1627            );
1628            match frame {
1629                Ok(frame) => {
1630                    if subscriber.frames.try_send(frame).is_ok() {
1631                        true
1632                    } else {
1633                        warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634                        if let Some(lagged) = subscriber.lagged.take() {
1635                            let _ = lagged.send(event.cursor.clone());
1636                        }
1637                        false
1638                    }
1639                }
1640                Err(error) => {
1641                    warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642                    false
1643                }
1644            }
1645        });
1646    }
1647
1648    fn subscribe(
1649        &self,
1650        connection_id: ConnectionId,
1651        corr: u64,
1652        version: u8,
1653        since: Option<SpawnCursor>,
1654        sink: FrameSink,
1655    ) -> Result<(), SpawnSubscribeRefusal> {
1656        let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657        let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658        {
1659            let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660            let replay = if let Some(since) = since {
1661                if since.daemon_incarnation != state.daemon_incarnation {
1662                    return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663                        current: state.daemon_incarnation.clone(),
1664                    });
1665                }
1666                if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667                    if since.seq < oldest.seq.saturating_sub(1) {
1668                        return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669                    }
1670                }
1671                state
1672                    .events
1673                    .iter()
1674                    .filter(|event| event.cursor.seq > since.seq)
1675                    .cloned()
1676                    .collect::<Vec<_>>()
1677            } else {
1678                Vec::new()
1679            };
1680            for event in replay {
1681                let body = serde_json::to_vec(&event)
1682                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683                let frame = Frame::build_with_version(
1684                    version,
1685                    FrameType::StreamData,
1686                    control_flags(),
1687                    0,
1688                    0,
1689                    corr,
1690                    body,
1691                )
1692                .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693                frames
1694                    .try_send(frame)
1695                    .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696            }
1697            state.subscribers.insert(
1698                (connection_id, corr),
1699                SpawnSubscriber {
1700                    version,
1701                    frames,
1702                    lagged: Some(lagged),
1703                },
1704            );
1705        }
1706        // The lagged terminal is sent here, by the forwarder, rather than by
1707        // the emitter: at the moment of the drop the subscriber's own channel
1708        // is full, and writing to the connection sink directly from the emitter
1709        // would put the Error AHEAD of the events still queued in that channel
1710        // (and the emitter holds the feed lock, so it cannot await the sink).
1711        // Dropping the subscriber drops the only sender, so `recv` drains every
1712        // queued event and then returns `None`; only then is the Error sent, so
1713        // the client sees each event it can keep, then the reason it was cut.
1714        // Cancel and connection removal drop the oneshot unsent, so they end
1715        // the stream with no Error.
1716        tokio::spawn(async move {
1717            while let Some(frame) = receiver.recv().await {
1718                if sink.send(frame).await.is_err() {
1719                    return;
1720                }
1721            }
1722            let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723                return;
1724            };
1725            match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726                Ok(frame) => {
1727                    let _ = sink.send(frame).await;
1728                }
1729                Err(error) => {
1730                    error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731                }
1732            }
1733        });
1734        Ok(())
1735    }
1736
1737    fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738        let Some(subscriber) = self
1739            .0
1740            .lock()
1741            .unwrap_or_else(|p| p.into_inner())
1742            .subscribers
1743            .remove(&(connection_id, corr))
1744        else {
1745            return false;
1746        };
1747        if let Ok(frame) = Frame::build_with_version(
1748            subscriber.version,
1749            FrameType::StreamEnd,
1750            control_flags(),
1751            0,
1752            0,
1753            corr,
1754            Vec::new(),
1755        ) {
1756            tokio::spawn(async move {
1757                let _ = subscriber.frames.send(frame).await;
1758            });
1759        }
1760        true
1761    }
1762
1763    fn remove_connection(&self, connection_id: ConnectionId) {
1764        self.0
1765            .lock()
1766            .unwrap_or_else(|p| p.into_inner())
1767            .subscribers
1768            .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769    }
1770
1771    #[cfg(any(test, feature = "test-support"))]
1772    fn set_capacity(&self, capacity: usize) {
1773        self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774    }
1775
1776    #[cfg(any(test, feature = "test-support"))]
1777    fn subscriber_count(&self) -> usize {
1778        self.0
1779            .lock()
1780            .unwrap_or_else(|p| p.into_inner())
1781            .subscribers
1782            .len()
1783    }
1784}
1785
1786/// Narrow process-liveness signal published by supervisors and consumed by passive liveness polls.
1787/// The terminal Error a lagged spawn subscriber receives after its queued events.
1788fn spawn_subscriber_lagged_frame(
1789    version: u8,
1790    corr: u64,
1791    first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793    let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794        code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795        message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796            .to_string(),
1797        detail: Some(serde_json::json!({
1798            "first_undelivered_cursor": first_undelivered
1799        })),
1800    })
1801    .map_err(|error| error.to_string())?;
1802    Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803        .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807    fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809    /// Whether the supervisor is replacing this module's process right now: an
1810    /// operator restart or reload, a health restart, or a crash respawn whose
1811    /// backoff is running. A module in that state is not live, but a consumer
1812    /// refused now should retry shortly rather than treat the target as gone.
1813    /// Stopped, failed, and disabled modules are not replacing.
1814    fn process_replacing(&self, _module_id: &str) -> bool {
1815        false
1816    }
1817}
1818
1819/// Shared process-liveness registry keyed by supervised `module_id`.
1820#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822    snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826    pub fn new() -> Self {
1827        Self::default()
1828    }
1829
1830    fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831        let mut snapshots = self
1832            .snapshots
1833            .lock()
1834            .unwrap_or_else(|poisoned| poisoned.into_inner());
1835        snapshots.insert(module_id, snapshot);
1836    }
1837
1838    fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839        let mut snapshots = self
1840            .snapshots
1841            .lock()
1842            .unwrap_or_else(|poisoned| poisoned.into_inner());
1843        let is_current = snapshots
1844            .get(module_id)
1845            .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846            .unwrap_or(false);
1847        if is_current {
1848            snapshots.remove(module_id);
1849        }
1850    }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854    fn process_live(&self, module_id: &str) -> Option<bool> {
1855        let snapshot = {
1856            let snapshots = self
1857                .snapshots
1858                .lock()
1859                .unwrap_or_else(|poisoned| poisoned.into_inner());
1860            snapshots.get(module_id).cloned()
1861        }?;
1862        let snapshot = snapshot
1863            .lock()
1864            .unwrap_or_else(|poisoned| poisoned.into_inner());
1865        Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866    }
1867
1868    fn process_replacing(&self, module_id: &str) -> bool {
1869        let Some(snapshot) = self
1870            .snapshots
1871            .lock()
1872            .unwrap_or_else(|poisoned| poisoned.into_inner())
1873            .get(module_id)
1874            .cloned()
1875        else {
1876            return false;
1877        };
1878        let snapshot = snapshot
1879            .lock()
1880            .unwrap_or_else(|poisoned| poisoned.into_inner());
1881        snapshot.enabled
1882            && match snapshot.state {
1883                ModuleState::Restarting => true,
1884                ModuleState::Draining => snapshot.draining_to_replace,
1885                ModuleState::Starting
1886                | ModuleState::Running
1887                | ModuleState::Unresponsive
1888                | ModuleState::Stopped
1889                | ModuleState::Failed
1890                | ModuleState::Disabled => false,
1891            }
1892    }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898    reached: tokio::sync::Notify,
1899    resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904    Spawn,
1905    Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910    deadline: Instant,
1911    kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918    start_requester: Option<Principal>,
1919    pending_activity_restart: bool,
1920    provenance_probe: ExecutableIdentityProbe,
1921    /// Restart operations hand their backoff to the loop, which keeps serving commands.
1922    scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923    /// A reload acknowledges completion only after its replacement registers.
1924    deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925    restart_policy: RestartPolicy,
1926    /// This module's RESOLVED drain budget: per-module config when present,
1927    /// else `default_drain_timeout`.
1928    drain_timeout: Duration,
1929    /// Shared with the status handle so the attested value changes atomically
1930    /// when a rescan updates the running drain policy.
1931    effective_drain_timeout: Arc<Mutex<Duration>>,
1932    /// The supervisor-wide fallback, kept so a configuration update that
1933    /// REMOVES the per-module override can re-resolve to it.
1934    default_drain_timeout: Duration,
1935    health: HealthConfig,
1936    connection_file_path: Option<PathBuf>,
1937    capture_logs_dir: Option<PathBuf>,
1938    forwarding: Option<Arc<ForwardingTable>>,
1939    /// The shared handle, so every spawn path (initial, restart, reload) records the
1940    /// reserved-module launch nonce the HELLO verifier checks against.
1941    supervisor_handle: Option<SupervisorHandle>,
1942    /// This module's stderr tail, shared with the [`SupervisedModule`] that answers
1943    /// status queries.
1944    ///
1945    /// One ring per module, held across every respawn. The lines explaining an exit
1946    /// are written BEFORE that exit, so a ring recreated per process would be empty
1947    /// exactly when it is asked for.
1948    stderr_ring: Arc<Mutex<StderrRing>>,
1949    terminal_ring: Arc<Mutex<TerminalRing>>,
1950    spawn_events: SpawnEventFeed,
1951    child_roster: ChildRoster,
1952    #[cfg(target_os = "linux")]
1953    cgroup_placement: Option<subc_cgroup::Placement>,
1954    #[cfg(test)]
1955    test_seed_stale_facts_before_enable_spawn: bool,
1956    #[cfg(test)]
1957    test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962    spec: ModuleSpec,
1963    health: HealthConfig,
1964}
1965
1966/// Shared daemon lookup table for supervised module handles.
1967///
1968/// Shared by clone between the [`Supervisor`] (which spawns processes) and the
1969/// channel-0 control handler (which verifies HELLOs and consumer route opens), so
1970/// launch nonces recorded at spawn are checked by the same daemon instance.
1971#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973    activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974    activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975    modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976    /// Module ids the supervisor has taken on. An id is added BEFORE the
1977    /// module's first process is spawned and removed only when the module
1978    /// leaves the roster (see [`Self::retire`]), so it is always a superset of
1979    /// the keys of `modules`.
1980    ///
1981    /// `modules` cannot answer "is this module configured?" on its own: a
1982    /// [`SupervisedModule`] only exists once its process has been spawned, and
1983    /// a fast child can connect, register, sync its scopes and ask about them
1984    /// before the supervisor has inserted it. Answering "not configured" in that
1985    /// gap makes scope admission refuse with the terminal "will never sync"
1986    /// instead of the retryable "has not synced yet".
1987    configured_ids: Arc<Mutex<HashSet<String>>>,
1988    spawn_events: SpawnEventFeed,
1989    /// The current expected launch nonce for each reserved module_id. Set when the
1990    /// supervisor spawns the reserved module; checked when a HELLO claims that id. A
1991    /// non-reserved module never has an entry here and is never nonce-checked.
1992    /// Reserved module ids and the nonce that authorizes their next HELLO.
1993    /// `None` means RESERVED WITH NO LEGITIMATE HOLDER — a reserved module that
1994    /// has never been spawned (e.g. configured `enabled: false`) — and refuses
1995    /// every HELLO. Before this was expressible, a reserved-but-never-spawned id
1996    /// had NO entry and admitted anyone: the reservation protected the nonce
1997    /// holder, not the NAME (found live by CKCRED's canary probe registering
1998    /// against a reserved scratch id).
1999    reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000    /// Module ids removed by an executed rescan and the unix-millisecond removal time.
2001    ///
2002    /// This is deliberately in-memory only: subc is state-free across daemon
2003    /// restarts, and the tombstone only explains the hours-after-removal window
2004    /// while this executing daemon is still alive. Do not persist it in a store.
2005    removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006    /// The current launch nonce for every supervised spawn. This is separate from
2007    /// reserved_nonces because consumer route.open attestation applies to all spawned
2008    /// modules, while HELLO id-squatting protection remains opt-in via `reserved`.
2009    spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010    /// Reserved namespace prefixes mapped to the supervised owner module whose
2011    /// current spawn nonce authorizes HELLO claims below the prefix.
2012    ///
2013    /// Per §2.6 this is not a same-user security barrier: a same-user process can
2014    /// read the key file and launch nonce env. Like exact reserved ids, it prevents
2015    /// accidental collisions and lower-trust processes from squatting protected
2016    /// namespaces.
2017    reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018    /// Blue/green swaps in progress, by module id. An entry exists from just
2019    /// before the candidate process is spawned until the swap has failed, or
2020    /// has cut over and the old process is gone. While it exists, HELLO for the
2021    /// id is gated on the swap token (see [`Self::swap_hello_admission`]) and
2022    /// consumer attestation accepts both processes' nonces.
2023    swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024    /// Told when a swap promotes its candidate; see [`SwapPromotionObserver`].
2025    promotion_observer: PromotionObserverSlot,
2026    /// Serializes module-set reconciliation with operator lifecycle commands. Without
2027    /// this daemon-wide ordering, a rescan could retire or update a module while a
2028    /// concurrent reload still held its old handle and launch specification.
2029    operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032/// Told when a swap has promoted its candidate to be the module's active
2033/// registration.
2034///
2035/// An ordinary HELLO runs the control plane's registration side effects (the
2036/// capability cache, the deny census, the requirement recompute) as it
2037/// registers. A swap candidate's HELLO does not, because it is not routable;
2038/// promotion is when those must run instead, and promotion happens in the
2039/// supervisor, which has no other way into the control handler.
2040pub(crate) trait SwapPromotionObserver: Send + Sync {
2041    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044/// The installed [`SwapPromotionObserver`], held weakly: the observer (the
2045/// control handler) owns this handle, so a strong reference back would be a
2046/// cycle that keeps both alive.
2047#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052        f.write_str("PromotionObserverSlot")
2053    }
2054}
2055
2056/// The nonces of one open swap.
2057#[derive(Debug, Clone)]
2058struct OpenSwap {
2059    /// The launch nonce minted for the candidate process. It is the swap
2060    /// token: the only thing that admits a HELLO into the candidate slot.
2061    candidate_nonce: String,
2062    /// The incumbent's launch nonce, captured when the swap opened. It is kept
2063    /// here because cutover moves the module's recorded spawn nonce to the
2064    /// candidate while the incumbent is still draining and its consumers are
2065    /// still attesting with this one.
2066    incumbent_nonce: Option<String>,
2067    /// Set once a HELLO has been admitted with the swap token, so the token
2068    /// admits one registration and cannot be replayed after cutover empties
2069    /// the candidate slot.
2070    candidate_admitted: bool,
2071}
2072
2073/// What the swap gate says about a HELLO. See
2074/// [`SupervisorHandle::swap_hello_admission`].
2075#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077    /// No swap is open for the id (or the HELLO carries the incumbent's own
2078    /// nonce); the ordinary gates decide.
2079    NotSwapping,
2080    /// The HELLO carries the swap token: register it into the candidate slot.
2081    Candidate,
2082    /// A swap is open and the HELLO carries a nonce the supervisor did not
2083    /// mint for this id, no nonce, or a token already used.
2084    Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089    Exact {
2090        module_id: String,
2091    },
2092    Prefix {
2093        prefix: String,
2094        owner_module_id: String,
2095    },
2096}
2097
2098impl SupervisorHandle {
2099    pub fn new() -> Self {
2100        Self::default()
2101    }
2102
2103    pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104        self.spawn_events.snapshot()
2105    }
2106
2107    pub(crate) fn subscribe_spawns(
2108        &self,
2109        connection_id: ConnectionId,
2110        corr: u64,
2111        version: u8,
2112        since: Option<SpawnCursor>,
2113        sink: FrameSink,
2114    ) -> Result<(), SpawnSubscribeRefusal> {
2115        self.spawn_events
2116            .subscribe(connection_id, corr, version, since, sink)
2117    }
2118
2119    pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120        self.spawn_events.cancel(connection_id, corr)
2121    }
2122
2123    pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124        self.spawn_events.remove_connection(connection_id);
2125    }
2126
2127    #[cfg(any(test, feature = "test-support"))]
2128    pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129        assert!(capacity > 0, "spawn event capacity must be non-zero");
2130        self.spawn_events.set_capacity(capacity);
2131    }
2132
2133    #[cfg(any(test, feature = "test-support"))]
2134    pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135        self.spawn_events.subscriber_count()
2136    }
2137
2138    /// Record the launch nonce from a supervised spawn, replacing any prior nonce so
2139    /// a respawn invalidates stale consumer identities.
2140    pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141        self.spawn_nonces
2142            .lock()
2143            .unwrap_or_else(|poisoned| poisoned.into_inner())
2144            .insert(module_id.to_string(), nonce);
2145    }
2146
2147    /// Record the launch nonce expected from the next HELLO for a reserved module,
2148    /// replacing any prior nonce (a respawn invalidates the previous one).
2149    pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150        self.reserved_nonces
2151            .lock()
2152            .unwrap_or_else(|poisoned| poisoned.into_inner())
2153            .insert(module_id.to_string(), Some(nonce));
2154    }
2155
2156    /// Record namespace prefixes owned by a supervised module.
2157    pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158        let mut owners = self
2159            .reserved_prefix_owners
2160            .lock()
2161            .unwrap_or_else(|poisoned| poisoned.into_inner());
2162        owners.retain(|_, owner| owner != owner_module_id);
2163        for prefix in prefixes {
2164            owners.insert(prefix.clone(), owner_module_id.to_string());
2165        }
2166    }
2167
2168    /// The launch nonce most recently minted for a module's spawn, if any.
2169    #[cfg(test)]
2170    pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171        self.spawn_nonces
2172            .lock()
2173            .unwrap_or_else(|poisoned| poisoned.into_inner())
2174            .get(module_id)
2175            .cloned()
2176    }
2177
2178    fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179        self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180        let spawn_nonce = self
2181            .spawn_nonces
2182            .lock()
2183            .unwrap_or_else(|poisoned| poisoned.into_inner())
2184            .get(&spec.module_id)
2185            .cloned();
2186        let mut reserved_nonces = self
2187            .reserved_nonces
2188            .lock()
2189            .unwrap_or_else(|poisoned| poisoned.into_inner());
2190        if spec.reserved {
2191            // `None` (no spawn nonce minted) is INSERTED, not skipped: a
2192            // reserved name whose module has never spawned has no legitimate
2193            // holder, and the entry's absence is what used to leave the name
2194            // open to the first claimant.
2195            reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196        }
2197        drop(reserved_nonces);
2198        // A later unreserved declaration must not silently unreserve an id that
2199        // was retained after its reserved configuration was removed. The explicit
2200        // release ceremony is the only operation that retires that gate.
2201        self.removal_tombstones
2202            .lock()
2203            .unwrap_or_else(|poisoned| poisoned.into_inner())
2204            .remove(&spec.module_id);
2205    }
2206
2207    /// Whether a HELLO claiming `module_id` is authorized. An exact reserved id is
2208    /// authorized only by its expected nonce; otherwise a matching reserved prefix
2209    /// is authorized by the owner module's current spawn nonce. Non-reserved ids
2210    /// with no matching prefix are always authorized.
2211    pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212        self.reserved_hello_verification(module_id, presented)
2213            .is_ok()
2214    }
2215
2216    /// Whether HELLO passed a nonce check that was actually required by reserved
2217    /// module policy, or the refusal explaining why a required check failed.
2218    pub(crate) fn reserved_hello_verification(
2219        &self,
2220        module_id: &str,
2221        presented: Option<&str>,
2222    ) -> Result<bool, ReservedHelloRejection> {
2223        let nonces = self
2224            .reserved_nonces
2225            .lock()
2226            .unwrap_or_else(|poisoned| poisoned.into_inner());
2227        if let Some(expected) = nonces.get(module_id) {
2228            // `None` = reserved with no legitimate holder: refuse every
2229            // presentation, because no process can hold a nonce that was never
2230            // minted. Only a real minted nonce admits, in constant time.
2231            let authorized = match expected {
2232                Some(expected) => {
2233                    presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2234                }
2235                None => false,
2236            };
2237            if authorized {
2238                return Ok(true);
2239            }
2240            return Err(ReservedHelloRejection::Exact {
2241                module_id: module_id.to_string(),
2242            });
2243        }
2244        drop(nonces);
2245
2246        let matched_prefix = self
2247            .reserved_prefix_owners
2248            .lock()
2249            .unwrap_or_else(|poisoned| poisoned.into_inner())
2250            .iter()
2251            .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2252            .max_by_key(|(prefix, _)| prefix.len())
2253            .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2254        let Some((prefix, owner_module_id)) = matched_prefix else {
2255            return Ok(false);
2256        };
2257
2258        let authorized = presented.is_some_and(|presented| {
2259            self.spawn_nonces
2260                .lock()
2261                .unwrap_or_else(|poisoned| poisoned.into_inner())
2262                .get(&owner_module_id)
2263                .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2264                // While the owner is being swapped, children started by
2265                // either of its two processes hold that process's nonce.
2266                || self.swap_nonce_matches(&owner_module_id, presented)
2267        });
2268        if authorized {
2269            Ok(true)
2270        } else {
2271            Err(ReservedHelloRejection::Prefix {
2272                prefix,
2273                owner_module_id,
2274            })
2275        }
2276    }
2277
2278    /// Whether a consumer connection proved it came from a daemon-spawned module.
2279    ///
2280    /// Absence of an expected spawn nonce is a hard failure: consumer_identity is
2281    /// accepted only for module ids the supervisor has spawned.
2282    pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2283        if presented.is_empty() {
2284            return false;
2285        }
2286        let nonces = self
2287            .spawn_nonces
2288            .lock()
2289            .unwrap_or_else(|poisoned| poisoned.into_inner());
2290        let current = nonces
2291            .get(module_id)
2292            .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2293        drop(nonces);
2294        // During a swap two processes of the module are alive, and a consumer
2295        // started by either one presents that process's nonce. Accepting only
2296        // the recorded one would fail the incumbent's consumers for the whole
2297        // overlap once cutover moves the record to the candidate.
2298        current || self.swap_nonce_matches(module_id, presented)
2299    }
2300
2301    /// Whether `presented` is either nonce of an open swap for `module_id`.
2302    fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2303        let swaps = self
2304            .swaps
2305            .lock()
2306            .unwrap_or_else(|poisoned| poisoned.into_inner());
2307        swaps.get(module_id).is_some_and(|swap| {
2308            constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2309                || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2310                    constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2311                })
2312        })
2313    }
2314
2315    /// Open a swap for `module_id` with the candidate's freshly minted nonce.
2316    /// Called before the candidate process exists.
2317    pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2318        let incumbent_nonce = self
2319            .spawn_nonces
2320            .lock()
2321            .unwrap_or_else(|poisoned| poisoned.into_inner())
2322            .get(module_id)
2323            .cloned();
2324        self.swaps
2325            .lock()
2326            .unwrap_or_else(|poisoned| poisoned.into_inner())
2327            .insert(
2328                module_id.to_string(),
2329                OpenSwap {
2330                    candidate_nonce,
2331                    incumbent_nonce,
2332                    candidate_admitted: false,
2333                },
2334            );
2335    }
2336
2337    /// Close the swap for `module_id`, releasing whichever nonce is no longer
2338    /// the module's recorded one.
2339    pub(crate) fn close_swap(&self, module_id: &str) {
2340        self.swaps
2341            .lock()
2342            .unwrap_or_else(|poisoned| poisoned.into_inner())
2343            .remove(module_id);
2344    }
2345
2346    /// Install the observer told about swap promotions, replacing any earlier
2347    /// one.
2348    pub(crate) fn set_swap_promotion_observer(
2349        &self,
2350        observer: std::sync::Weak<dyn SwapPromotionObserver>,
2351    ) {
2352        *self
2353            .promotion_observer
2354            .0
2355            .lock()
2356            .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2357    }
2358
2359    /// Tell the installed observer, if it is still alive, that a swap promoted
2360    /// `registration`.
2361    fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2362        let observer = self
2363            .promotion_observer
2364            .0
2365            .lock()
2366            .unwrap_or_else(|poisoned| poisoned.into_inner())
2367            .as_ref()
2368            .and_then(std::sync::Weak::upgrade);
2369        if let Some(observer) = observer {
2370            observer.swap_promoted(registration);
2371        }
2372    }
2373
2374    /// Whether a swap is open for `module_id`.
2375    pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2376        self.swaps
2377            .lock()
2378            .unwrap_or_else(|poisoned| poisoned.into_inner())
2379            .contains_key(module_id)
2380    }
2381
2382    /// Make the candidate's nonce the module's recorded spawn nonce, as a plain
2383    /// respawn would, once cutover has made the candidate the module's process.
2384    /// The swap stays open so the incumbent's nonce keeps attesting until the
2385    /// incumbent has drained and exited.
2386    fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2387        let candidate_nonce = self
2388            .swaps
2389            .lock()
2390            .unwrap_or_else(|poisoned| poisoned.into_inner())
2391            .get(module_id)
2392            .map(|swap| swap.candidate_nonce.clone());
2393        let Some(nonce) = candidate_nonce else {
2394            return;
2395        };
2396        self.set_spawn_nonce(module_id, nonce.clone());
2397        if reserved {
2398            self.set_reserved_nonce(module_id, nonce);
2399        }
2400    }
2401
2402    /// The swap gate for a HELLO claiming `module_id`.
2403    ///
2404    /// This runs BEFORE the reserved-module gate. A reserved module's candidate
2405    /// presents the candidate nonce, which the reserved gate (holding the
2406    /// incumbent's nonce) would refuse as `reserved_module` before swap
2407    /// admission was ever reached. And it applies to unreserved ids too: for an
2408    /// unreserved id the only thing that ever stopped a second process claiming
2409    /// a live id was the `duplicate_module_id` refusal, which is exactly the
2410    /// refusal a swap lifts for its candidate.
2411    ///
2412    /// The incumbent's own nonce falls through to the ordinary gates, which
2413    /// treat it as they always have (a live incumbent is refused as a
2414    /// duplicate). Anything else while a swap is open is refused, including an
2415    /// absent nonce.
2416    pub(crate) fn swap_hello_admission(
2417        &self,
2418        module_id: &str,
2419        presented: Option<&str>,
2420    ) -> SwapHelloAdmission {
2421        let swaps = self
2422            .swaps
2423            .lock()
2424            .unwrap_or_else(|poisoned| poisoned.into_inner());
2425        let Some(swap) = swaps.get(module_id) else {
2426            return SwapHelloAdmission::NotSwapping;
2427        };
2428        let Some(presented) = presented else {
2429            return SwapHelloAdmission::Refused;
2430        };
2431        if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2432            return if swap.candidate_admitted {
2433                SwapHelloAdmission::Refused
2434            } else {
2435                SwapHelloAdmission::Candidate
2436            };
2437        }
2438        if swap
2439            .incumbent_nonce
2440            .as_deref()
2441            .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2442        {
2443            return SwapHelloAdmission::NotSwapping;
2444        }
2445        SwapHelloAdmission::Refused
2446    }
2447
2448    /// Record that the swap token has registered a candidate, so it admits no
2449    /// second HELLO.
2450    pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2451        if let Some(swap) = self
2452            .swaps
2453            .lock()
2454            .unwrap_or_else(|poisoned| poisoned.into_inner())
2455            .get_mut(module_id)
2456        {
2457            swap.candidate_admitted = true;
2458        }
2459    }
2460
2461    /// Test/support lookup for the current launch nonce of a supervised spawn.
2462    pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2463        self.spawn_nonces
2464            .lock()
2465            .unwrap_or_else(|poisoned| poisoned.into_inner())
2466            .get(module_id)
2467            .cloned()
2468    }
2469
2470    /// Test/support lookup for the HELLO-gating nonce of a reserved module.
2471    pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2472        self.reserved_nonces
2473            .lock()
2474            .unwrap_or_else(|poisoned| poisoned.into_inner())
2475            .get(module_id)
2476            .cloned()
2477            .flatten()
2478    }
2479
2480    pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2481        // Normally already marked before the process was spawned; marking here
2482        // too keeps `configured_ids` a superset of the roster for any caller
2483        // that inserts a module directly.
2484        self.mark_configured(module.module_id());
2485        let mut modules = self
2486            .modules
2487            .lock()
2488            .unwrap_or_else(|poisoned| poisoned.into_inner());
2489        modules.insert(module.module_id().to_string(), module)
2490    }
2491
2492    /// Record that the supervisor has taken on `module_id`. Called before the
2493    /// module's first process is spawned, so that by the time that process can
2494    /// register, [`Self::is_configured`] already answers true.
2495    fn mark_configured(&self, module_id: &str) {
2496        self.configured_ids
2497            .lock()
2498            .unwrap_or_else(|poisoned| poisoned.into_inner())
2499            .insert(module_id.to_string());
2500    }
2501
2502    /// Undo [`Self::mark_configured`] for a module whose first spawn failed
2503    /// before it was ever put on the roster. A module already on the roster
2504    /// keeps its mark: only [`Self::retire`] takes a rostered module off.
2505    fn unmark_configured_unless_rostered(&self, module_id: &str) {
2506        let modules = self
2507            .modules
2508            .lock()
2509            .unwrap_or_else(|poisoned| poisoned.into_inner());
2510        if !modules.contains_key(module_id) {
2511            self.configured_ids
2512                .lock()
2513                .unwrap_or_else(|poisoned| poisoned.into_inner())
2514                .remove(module_id);
2515        }
2516    }
2517
2518    /// Whether `module_id` is a module this daemon supervises: on the roster,
2519    /// or about to be (its process is being spawned right now).
2520    ///
2521    /// This, not `get(..).is_some()`, is what "the owner is configured" means
2522    /// for scopes: a supervised module's process can register and sync before
2523    /// [`Self::get`] can return it, and in that window it is still a module
2524    /// that will sync, not one that never will.
2525    pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2526        self.configured_ids
2527            .lock()
2528            .unwrap_or_else(|poisoned| poisoned.into_inner())
2529            .contains(module_id)
2530    }
2531
2532    pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2533        let modules = self
2534            .modules
2535            .lock()
2536            .unwrap_or_else(|poisoned| poisoned.into_inner());
2537        modules.get(module_id).cloned()
2538    }
2539
2540    pub(crate) fn record_late_health_answer(
2541        &self,
2542        module_id: &str,
2543        latency_ms: u64,
2544    ) -> Result<bool, SuperviseError> {
2545        let Some(module) = self.get(module_id) else {
2546            return Ok(false);
2547        };
2548        update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2549            state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2550            state.health.last_late_answer_latency_ms = Some(latency_ms);
2551            // A late answer is an answer: the module served the probe, just past
2552            // the deadline. Leaving the miss streak in place while logging
2553            // "proves the module is alive" is how a CPU-starved module that
2554            // answers every probe a few seconds late still marches to the
2555            // threshold and gets killed — the exact kill class `NoAnswer` is
2556            // excluded from `is_proof_of_death` to prevent. Slow-but-answering
2557            // is degradation, and degradation reports; it does not restart.
2558            state.health.consecutive_failures = 0;
2559        })?;
2560        Ok(true)
2561    }
2562
2563    /// Arm the one-shot marker for the module process that this caller
2564    /// deliberately initiated severance against. Generic connection teardown
2565    /// must not call this:
2566    /// a surviving process would otherwise retain an exemption for a later
2567    /// genuine crash.
2568    pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2569        let Some(module) = self.get(module_id) else {
2570            return Ok(false);
2571        };
2572        let snapshot = lock_snapshot(&module.inner.snapshot)?;
2573        let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2574            return Ok(false);
2575        };
2576        drop(snapshot);
2577        module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2578    }
2579
2580    pub fn list(&self) -> Vec<SupervisedModule> {
2581        let modules = self
2582            .modules
2583            .lock()
2584            .unwrap_or_else(|poisoned| poisoned.into_inner());
2585        let mut modules = modules.values().cloned().collect::<Vec<_>>();
2586        modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2587        modules
2588    }
2589
2590    pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2591        self.activity_images
2592            .lock()
2593            .unwrap_or_else(|p| p.into_inner())
2594            .remove(module_id);
2595        self.spawn_nonces
2596            .lock()
2597            .unwrap_or_else(|poisoned| poisoned.into_inner())
2598            .remove(module_id);
2599        self.close_swap(module_id);
2600        let mut reserved_nonces = self
2601            .reserved_nonces
2602            .lock()
2603            .unwrap_or_else(|poisoned| poisoned.into_inner());
2604        if reserved_nonces.contains_key(module_id) {
2605            // The old nonce must die with the removed process, but the exact-id
2606            // gate remains until an operator explicitly releases it.
2607            reserved_nonces.insert(module_id.to_string(), None);
2608        }
2609        drop(reserved_nonces);
2610        self.reserved_prefix_owners
2611            .lock()
2612            .unwrap_or_else(|poisoned| poisoned.into_inner())
2613            .retain(|_, owner| owner != module_id);
2614        let removed = self
2615            .modules
2616            .lock()
2617            .unwrap_or_else(|poisoned| poisoned.into_inner())
2618            .remove(module_id);
2619        self.configured_ids
2620            .lock()
2621            .unwrap_or_else(|poisoned| poisoned.into_inner())
2622            .remove(module_id);
2623        removed
2624    }
2625
2626    /// Remember a module removed by a non-preview rescan so route.open can
2627    /// distinguish that intentional removal from an unknown id.
2628    pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2629        self.removal_tombstones
2630            .lock()
2631            .unwrap_or_else(|poisoned| poisoned.into_inner())
2632            .insert(module_id.to_string(), unix_ms_now());
2633    }
2634
2635    /// Return how long ago a rescan removed this module in milliseconds.
2636    pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2637        self.removal_tombstones
2638            .lock()
2639            .unwrap_or_else(|poisoned| poisoned.into_inner())
2640            .get(module_id)
2641            .copied()
2642            .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2643    }
2644
2645    /// Retire a reserved-id gate only after its module has left supervision.
2646    ///
2647    /// A retained gate has no live nonce (`None`), so releasing any other entry
2648    /// would weaken a currently configured or otherwise active reservation.
2649    pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2650        if self.get(module_id).is_some() {
2651            return false;
2652        }
2653        let mut reserved_nonces = self
2654            .reserved_nonces
2655            .lock()
2656            .unwrap_or_else(|poisoned| poisoned.into_inner());
2657        if !matches!(reserved_nonces.get(module_id), Some(None)) {
2658            return false;
2659        }
2660        reserved_nonces.remove(module_id);
2661        true
2662    }
2663
2664    pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2665        Arc::clone(&self.operation_lock)
2666    }
2667}
2668
2669/// Process supervisor for subc-owned singleton modules.
2670#[derive(Debug, Clone)]
2671pub struct Supervisor {
2672    registry: Arc<Registry>,
2673    restart_policy: RestartPolicy,
2674    drain_timeout: Duration,
2675    connection_file_path: Option<PathBuf>,
2676    capture_logs_dir: Option<PathBuf>,
2677    forwarding: Option<Arc<ForwardingTable>>,
2678    process_liveness: Arc<SupervisorProcessLiveness>,
2679    supervisor_handle: Option<SupervisorHandle>,
2680    health: HealthConfig,
2681    daemon_start_clock: crate::clock::StartClock,
2682    terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2683    spawn_events: SpawnEventFeed,
2684    provenance_probe: ExecutableIdentityProbe,
2685    /// Every process spawned through this supervisor (and its clones) and not
2686    /// yet reaped, so daemon shutdown can end them.
2687    child_roster: ChildRoster,
2688    #[cfg(target_os = "linux")]
2689    cgroup_placement: Option<subc_cgroup::Placement>,
2690    #[cfg(test)]
2691    test_after_first_spawn: AfterFirstSpawnHook,
2692}
2693
2694/// A roster entry whose monitor and first process have not started yet.
2695/// Keeping the command receiver here lets discovery serve status immediately
2696/// while start/stop requests wait until bootstrap has published the connection file.
2697pub(crate) struct RegisteredConfiguredModule {
2698    spec: ModuleSpec,
2699    runtime: SupervisorRuntimeConfig,
2700    module: SupervisedModule,
2701    commands: mpsc::Receiver<SupervisorCommand>,
2702}
2703
2704/// Test-only hook run on the path that takes on a new module, right after its
2705/// first `spawn_child` returns (the process exists and could already be
2706/// registering) and before that process is handed to the module's supervise
2707/// loop and put on the roster. Lets a test observe what a fast child would see
2708/// in that window without racing a real one.
2709#[cfg(test)]
2710#[derive(Clone, Default)]
2711struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2712
2713#[cfg(test)]
2714type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2715
2716#[cfg(test)]
2717impl fmt::Debug for AfterFirstSpawnHook {
2718    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2719        f.write_str("AfterFirstSpawnHook")
2720    }
2721}
2722
2723#[cfg(test)]
2724impl AfterFirstSpawnHook {
2725    fn run(&self, module_id: &str) {
2726        if let Some(hook) = &self.0 {
2727            hook(module_id);
2728        }
2729    }
2730}
2731
2732impl Supervisor {
2733    #[cfg(test)]
2734    pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2735        let supervisor = Self::new(registry, policy);
2736        #[cfg(target_os = "macos")]
2737        let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2738        supervisor
2739    }
2740    /// Verify the trampoline once when configured. Missing private OS support
2741    /// refuses every macOS launch by name but does not stop the daemon's control
2742    /// server. Embedders must explicitly provide a binary with the subc-os hidden
2743    /// entry point; the library must not exec an arbitrary hosting program.
2744    pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2745        let path = path.into();
2746        #[cfg(target_os = "macos")]
2747        {
2748            let result = probe_privacy_trampoline(&path).map(|()| path);
2749            if let Err(cause) = &result {
2750                error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2751            }
2752            self.child_roster.set_privacy_trampoline(result);
2753        }
2754        #[cfg(not(target_os = "macos"))]
2755        let _ = path;
2756        self
2757    }
2758    /// The flag [`Self::begin_daemon_shutdown`] sets, for the control handler's
2759    /// capability-requirement log to read.
2760    pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2761        self.child_roster.shutdown_flag()
2762    }
2763
2764    /// The first step of an announced daemon shutdown, before the notice and
2765    /// before any connection is closed.
2766    ///
2767    /// Sets the daemon-shutdown flag first: from here on no module is
2768    /// respawned (crash restart, operator restart, or swap), and every child
2769    /// exit is recorded as `daemon_shutdown` rather than as a crash, whether
2770    /// the module exits on the EOF this shutdown gives it or is signalled by a
2771    /// service manager that kills the whole cgroup. Then writes the journal's
2772    /// shutdown marker, which records the instant and closes this daemon
2773    /// incarnation's stretch of the journal.
2774    #[cfg(unix)]
2775    pub(crate) fn begin_daemon_shutdown(&self) {
2776        self.child_roster.close();
2777        if let Some(journal) = &self.terminal_journal {
2778            journal.stamp_shutdown();
2779        }
2780    }
2781
2782    /// Announce a cut while established connections can still carry replies.
2783    /// These budgets promise notice and a bounded wait, not child completion;
2784    /// they are local policy, not an estimate of launchd's unknown kill ceiling.
2785    #[cfg(unix)]
2786    pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2787        const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2788        const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2789        let Some(forwarding) = &self.forwarding else {
2790            return Ok(());
2791        };
2792        let modules = forwarding
2793            .begin_daemon_drain()
2794            .map_err(SuperviseError::Forwarding)?;
2795        let deadline_ms =
2796            unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2797        let mut notices = tokio::task::JoinSet::new();
2798        let mut drains = Vec::new();
2799        for (module_id, endpoint) in modules {
2800            let Some(target) = forwarding
2801                .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2802                .map_err(SuperviseError::Forwarding)?
2803            else {
2804                continue;
2805            };
2806            let routes = forwarding
2807                .endpoint_routes(target.endpoint)
2808                .map_err(SuperviseError::Forwarding)?;
2809            // Restart allows deployed consumers to reopen after the new daemon
2810            // appears. The wire reason stays `restart`; what tells a daemon cut
2811            // apart from a module restart afterwards is the terminal record
2812            // itself, whose disposition is `daemon_shutdown` for every exit
2813            // observed once `begin_daemon_shutdown` has run.
2814            let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2815                reason: RouteCloseReason::Restart,
2816                deadline_ms,
2817            })
2818            .expect("module draining serializes");
2819            let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2820            let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2821            for route in routes {
2822                let client = route.goodbye_target;
2823                if let Some((_, channels)) = clients
2824                    .iter_mut()
2825                    .find(|(existing, _)| existing.connection_id == client.connection_id)
2826                {
2827                    channels.push(client.channel);
2828                } else {
2829                    let channel = client.channel;
2830                    clients.push((client, vec![channel]));
2831                }
2832            }
2833            for (client, mut channels) in clients {
2834                channels.sort_unstable();
2835                channels.dedup();
2836                let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2837                    module_id: module_id.clone(),
2838                    channels,
2839                    reason: RouteCloseReason::Restart,
2840                })
2841                .expect("route closing serializes");
2842                recipients.push((client.sink, client.negotiated_ver, closing));
2843            }
2844            for (sink, version, body) in recipients {
2845                notices.spawn(async move {
2846                    let frame = Frame::build_with_version(
2847                        version,
2848                        FrameType::Push,
2849                        control_flags(),
2850                        0,
2851                        0,
2852                        0,
2853                        body,
2854                    )
2855                    .expect("bounded lifecycle notice frame builds");
2856                    sink.send_flushed(frame).await
2857                });
2858            }
2859            let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2860            drains.push((module_id, target.endpoint, gauges));
2861        }
2862        // A quiet forwarding table is not proof that queued notices reached the
2863        // socket. Wait for writer flush acknowledgements before testing quiescence.
2864        let notice_deadline = Instant::now() + NOTICE_BUDGET;
2865        while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2866            if !matches!(result, Ok(Ok(()))) {
2867                warn!(?result, "daemon shutdown notice delivery failed");
2868            }
2869        }
2870        notices.abort_all();
2871        let deadline = Instant::now() + DRAIN_BUDGET;
2872        let mut waits = tokio::task::JoinSet::new();
2873        for (module_id, endpoint, gauges) in drains {
2874            let forwarding = Arc::clone(forwarding);
2875            let mut runtime = self.runtime_config();
2876            runtime.health.cadence = Duration::from_millis(100);
2877            waits.spawn(async move {
2878                let wait_result = wait_for_forwarding_quiescence(
2879                    &forwarding,
2880                    &module_id,
2881                    &runtime,
2882                    endpoint,
2883                    deadline,
2884                    &gauges,
2885                    DrainScope::Endpoint(endpoint),
2886                    &mut DrainGaugeTrace::new(&gauges),
2887                )
2888                .await;
2889                // Store the measurement before the task returns: quiescence
2890                // may be reached just as the deadline stops the join loop from
2891                // collecting results. Unfinished or failed waits report false.
2892                forwarding
2893                    .record_daemon_drain_result(
2894                        endpoint,
2895                        drained_after_quiescence_wait(&wait_result),
2896                    )
2897                    .map_err(SuperviseError::Forwarding)?;
2898                wait_result
2899            });
2900        }
2901        while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2902            if !matches!(result, Ok(Ok(true))) {
2903                warn!(?result, "daemon shutdown drain did not reach quiescence");
2904            }
2905        }
2906        Ok(())
2907    }
2908
2909    /// The last step of an announced daemon shutdown, after the notice and the
2910    /// drain: send every registered module a module GOODBYE, the same planned
2911    /// stop signal `ck module stop` gives, then close every connection so each
2912    /// subc module sees EOF and starts its own teardown, then end every
2913    /// supervised child that has not exited
2914    /// by its own deadline (its drain budget, capped). Modules lead their own
2915    /// process groups, so a
2916    /// service manager's group kill no longer reaches them; without this a
2917    /// child that does not stop on EOF (every `protocol: "none"` child, which
2918    /// has no connection) would outlive the daemon. Every wait is bounded (see
2919    /// `child_roster`), and `escalate` resolving (a second SIGTERM) cuts them.
2920    #[cfg(unix)]
2921    pub(crate) async fn end_children_for_daemon_shutdown(
2922        &self,
2923        already_escalated: bool,
2924        escalate: impl std::future::Future<Output = ()>,
2925    ) {
2926        tokio::pin!(escalate);
2927        let mut escalated = already_escalated;
2928        if let Some(forwarding) = &self.forwarding {
2929            let reason = CloseReason::new(
2930                "daemon_shutdown",
2931                "the daemon is exiting after its shutdown notice and drain",
2932            );
2933            if escalated {
2934                // The operator asked to stop waiting: queue the GOODBYEs but
2935                // do not wait for them to be written.
2936                send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2937            } else {
2938                tokio::select! {
2939                    biased;
2940                    _ = escalate.as_mut() => {
2941                        info!("second SIGTERM: abandoning module GOODBYE delivery");
2942                        escalated = true;
2943                    }
2944                    _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2945                }
2946            }
2947            let closed = forwarding.close_all_connections(&reason);
2948            debug!(closed, "closed established connections for daemon shutdown");
2949        }
2950        // If the second SIGTERM arrived during GOODBYE delivery, `escalate` has
2951        // already completed and must not be polled again; the child shutdown
2952        // wait is told it is escalated and gets a future that never fires.
2953        let escalated_here = escalated && !already_escalated;
2954        let remaining_escalate = async move {
2955            if escalated_here {
2956                std::future::pending::<()>().await;
2957            } else {
2958                escalate.await;
2959            }
2960        };
2961        crate::child_roster::end_children_for_daemon_shutdown(
2962            &self.child_roster,
2963            escalated,
2964            remaining_escalate,
2965        )
2966        .await;
2967    }
2968
2969    pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2970        Self {
2971            registry,
2972            restart_policy,
2973            drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2974            connection_file_path: None,
2975            capture_logs_dir: None,
2976            forwarding: None,
2977            process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2978            supervisor_handle: None,
2979            health: HealthConfig::default(),
2980            daemon_start_clock: crate::clock::StartClock::capture(),
2981            terminal_journal: None,
2982            spawn_events: SpawnEventFeed::default(),
2983            provenance_probe: ExecutableIdentityProbe::default(),
2984            child_roster: ChildRoster::default(),
2985            #[cfg(target_os = "linux")]
2986            cgroup_placement: None,
2987            #[cfg(test)]
2988            test_after_first_spawn: AfterFirstSpawnHook::default(),
2989        }
2990    }
2991
2992    pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2993        self.drain_timeout = drain_timeout;
2994        self
2995    }
2996
2997    /// The drain budget a module gets when its config entry sets none.
2998    pub(crate) fn default_drain_timeout(&self) -> Duration {
2999        self.drain_timeout
3000    }
3001
3002    pub fn with_process_liveness(
3003        mut self,
3004        process_liveness: Arc<SupervisorProcessLiveness>,
3005    ) -> Self {
3006        self.process_liveness = process_liveness;
3007        self
3008    }
3009
3010    pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3011        self.connection_file_path = Some(connection_file_path.into());
3012        self
3013    }
3014
3015    /// Enables daemon-owned capture files for supervised stdout and stderr.
3016    pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3017        self.capture_logs_dir = Some(logs_dir.into());
3018        self
3019    }
3020
3021    /// Names this daemon lifetime in spawn events, independently of whether a
3022    /// terminal journal is configured.
3023    pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3024        // A millisecond start stamp can repeat after clock rollback or a rapid
3025        // restart. Use the connection file's random daemon_id instead: it already
3026        // identifies this daemon lifetime independently of the wall clock.
3027        self.spawn_events.configure_incarnation(daemon_incarnation);
3028        self
3029    }
3030
3031    /// Enables best-effort history shared by every supervised module. Without
3032    /// it, terminal history is kept only in each module's in-memory ring.
3033    pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3034        let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3035        this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3036            path,
3037            daemon_incarnation,
3038        )));
3039        this
3040    }
3041
3042    pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3043        self.forwarding = Some(forwarding);
3044        self
3045    }
3046
3047    pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3048        self.spawn_events = supervisor_handle.spawn_events.clone();
3049        self.supervisor_handle = Some(supervisor_handle);
3050        self
3051    }
3052
3053    pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3054        self.health = health;
3055        self
3056    }
3057
3058    /// Keeps a record of every live child at `path`, rewritten on each spawn and
3059    /// reap, for the orphan sweep a later daemon runs at boot. Without it no
3060    /// record is kept.
3061    pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3062        self.child_roster.record_to(path.into());
3063        self
3064    }
3065
3066    #[cfg(target_os = "linux")]
3067    pub fn with_cgroup_placement(
3068        mut self,
3069        cgroup_placement: Option<subc_cgroup::Placement>,
3070    ) -> Self {
3071        self.cgroup_placement = cgroup_placement;
3072        self
3073    }
3074
3075    /// Spawn `spec.program` and start monitoring it.
3076    ///
3077    /// The child is expected to parse `--subc <connection-file-path>`, read the
3078    /// TCP+key connection file, authenticate to the already-running listener, and
3079    /// register with channel-0 `HELLO` using `spec.module_id` as its manifest id.
3080    pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3081        validate_spec(&spec)?;
3082        self.establish_identity(&spec);
3083
3084        let runtime = self.runtime_config();
3085        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3086        let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3087        let spawned = spawn_child(
3088            &spec,
3089            runtime.connection_file_path.as_deref(),
3090            self.supervisor_handle.as_ref(),
3091            &runtime.stderr_ring,
3092            runtime.capture_logs_dir.as_deref(),
3093            &runtime.child_roster,
3094            #[cfg(target_os = "linux")]
3095            runtime.cgroup_placement.as_ref(),
3096        );
3097        #[cfg(test)]
3098        self.test_after_first_spawn.run(&spec.module_id);
3099        let child = match spawned {
3100            Ok(child) => child,
3101            Err(err) => {
3102                // Unlike the configured paths, a failed `spawn` leaves nothing
3103                // on the roster, so the module must not stay marked configured.
3104                self.abandon_unrostered(&spec.module_id);
3105                return Err(err);
3106            }
3107        };
3108        self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3109        if let Some(spawn) = activity_spawn {
3110            spawn.complete(&snapshot);
3111        }
3112
3113        Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3114    }
3115
3116    /// Make `spec`'s module count as configured, with its identity gates
3117    /// (reserved id, reserved prefixes) in place, BEFORE any process of it
3118    /// exists.
3119    ///
3120    /// Every path that takes on a new module calls this before `spawn_child`.
3121    /// The order is the point: the child can connect, register, sync its
3122    /// scopes and ask about them as soon as it is spawned, and the module is
3123    /// only put on the roster after `spawn_child` returns. Were the mark set
3124    /// with the roster entry, a fast child would see its own owner reported
3125    /// as not configured, and a scoped `route.open` in that window would be
3126    /// refused as terminal `scope_not_live` ("will never sync") instead of
3127    /// retryable `scope_not_synced`.
3128    fn establish_identity(&self, spec: &ModuleSpec) {
3129        if let Some(supervisor_handle) = &self.supervisor_handle {
3130            supervisor_handle.apply_identity_configuration(spec);
3131            supervisor_handle.mark_configured(&spec.module_id);
3132        }
3133    }
3134
3135    /// Take back [`Self::establish_identity`]'s configured mark when the
3136    /// module will not be put on the roster after all.
3137    fn abandon_unrostered(&self, module_id: &str) {
3138        if let Some(supervisor_handle) = &self.supervisor_handle {
3139            supervisor_handle.unmark_configured_unless_rostered(module_id);
3140        }
3141    }
3142
3143    /// Record a freshly spawned first process as running. On failure the
3144    /// module never reaches the roster, so its configured mark is taken back.
3145    fn mark_first_process_running(
3146        &self,
3147        spec: &ModuleSpec,
3148        runtime: &SupervisorRuntimeConfig,
3149        snapshot: &SharedSnapshot,
3150        child: &SupervisedChild,
3151    ) -> Result<(), SuperviseError> {
3152        if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3153            self.abandon_unrostered(&spec.module_id);
3154            return Err(err);
3155        }
3156        self.process_liveness
3157            .track(spec.module_id.clone(), Arc::clone(snapshot));
3158        Ok(())
3159    }
3160
3161    /// Start supervising a module declared in daemon configuration.
3162    ///
3163    /// Unlike [`Self::spawn`], this records disabled modules and immediate spawn
3164    /// failures in the supervisor handle so operator-facing `supervisor.list`
3165    /// reflects every configured module while daemon startup continues.
3166    pub fn supervise_configured(
3167        &self,
3168        spec: ModuleSpec,
3169        enabled: bool,
3170    ) -> Result<SupervisedModule, SuperviseError> {
3171        self.supervise_configured_requested(
3172            spec,
3173            enabled,
3174            self.health.clone(),
3175            None,
3176            self.restart_policy,
3177            None,
3178        )
3179    }
3180
3181    /// Supervise a configured module with its own health, drain, and crash
3182    /// budget. The restart policy is per-module because the config file is:
3183    /// `modules.<id>.restart` resolves to a full policy at parse time, and a
3184    /// module that is expensive to restart should not be forced onto the same
3185    /// budget as one that is cheap.
3186    pub fn supervise_configured_with_health(
3187        &self,
3188        spec: ModuleSpec,
3189        enabled: bool,
3190        health: HealthConfig,
3191        drain_timeout_ms: Option<u64>,
3192        restart_policy: RestartPolicy,
3193    ) -> Result<SupervisedModule, SuperviseError> {
3194        self.supervise_configured_requested(
3195            spec,
3196            enabled,
3197            health,
3198            drain_timeout_ms,
3199            restart_policy,
3200            None,
3201        )
3202    }
3203
3204    pub(crate) fn supervise_configured_requested(
3205        &self,
3206        spec: ModuleSpec,
3207        enabled: bool,
3208        health: HealthConfig,
3209        drain_timeout_ms: Option<u64>,
3210        restart_policy: RestartPolicy,
3211        requester: Option<Principal>,
3212    ) -> Result<SupervisedModule, SuperviseError> {
3213        let registered = self.register_configured_with_health(
3214            spec,
3215            enabled,
3216            health,
3217            drain_timeout_ms,
3218            restart_policy,
3219        )?;
3220        self.start_registered_configured(registered, requester)
3221    }
3222
3223    /// Record a configured module as starting or disabled, without launching a
3224    /// process or running its monitor. Bootstrap registers the entire roster
3225    /// before publishing discovery, then activates it once children can read
3226    /// the connection file passed in their `--subc` argument.
3227    pub(crate) fn register_configured_with_health(
3228        &self,
3229        spec: ModuleSpec,
3230        enabled: bool,
3231        health: HealthConfig,
3232        drain_timeout_ms: Option<u64>,
3233        restart_policy: RestartPolicy,
3234    ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3235        validate_spec(&spec)?;
3236        self.establish_identity(&spec);
3237
3238        let mut runtime = self.runtime_config();
3239        runtime.health = health;
3240        runtime.restart_policy = restart_policy;
3241        if let Some(ms) = drain_timeout_ms {
3242            runtime.drain_timeout = Duration::from_millis(ms);
3243            *runtime
3244                .effective_drain_timeout
3245                .lock()
3246                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3247        }
3248        let snapshot = if enabled {
3249            SupervisorSnapshot::starting()
3250        } else {
3251            SupervisorSnapshot::disabled()
3252        };
3253        let snapshot = {
3254            let mut snapshot = snapshot;
3255            if enabled {
3256                if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3257                {
3258                    snapshot.waiting_for_previous_pid = Some(pid);
3259                    snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3260                }
3261            }
3262            snapshot
3263        };
3264        let (module, commands) =
3265            self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3266        Ok(RegisteredConfiguredModule {
3267            spec,
3268            runtime,
3269            module,
3270            commands,
3271        })
3272    }
3273
3274    /// Activate a previously registered module. Queued control requests are
3275    /// served by the monitor after the initial configured spawn has settled.
3276    pub(crate) fn start_registered_configured(
3277        &self,
3278        registered: RegisteredConfiguredModule,
3279        requester: Option<Principal>,
3280    ) -> Result<SupervisedModule, SuperviseError> {
3281        let RegisteredConfiguredModule {
3282            spec,
3283            mut runtime,
3284            module,
3285            commands,
3286        } = registered;
3287        runtime.start_requester = requester;
3288        let snapshot = &module.inner.snapshot;
3289        let should_spawn = {
3290            let state = lock_snapshot(snapshot)?;
3291            state.enabled && state.waiting_for_previous_pid.is_none()
3292        };
3293        let child = if should_spawn {
3294            let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3295            let spawned = spawn_child(
3296                &spec,
3297                runtime.connection_file_path.as_deref(),
3298                self.supervisor_handle.as_ref(),
3299                &runtime.stderr_ring,
3300                runtime.capture_logs_dir.as_deref(),
3301                &runtime.child_roster,
3302                #[cfg(target_os = "linux")]
3303                runtime.cgroup_placement.as_ref(),
3304            );
3305            #[cfg(test)]
3306            self.test_after_first_spawn.run(&spec.module_id);
3307            match spawned {
3308                Ok(child) => {
3309                    self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3310                    if let Some(spawn) = activity_spawn {
3311                        spawn.complete(snapshot);
3312                    }
3313                    Some(child)
3314                }
3315                Err(err) => {
3316                    if runtime.health.critical {
3317                        error!(
3318                            module_id = %spec.module_id,
3319                            program = %spec.program.display(),
3320                            error = %err,
3321                            "critical configured module failed to spawn; marking failed and alerting"
3322                        );
3323                    } else {
3324                        error!(
3325                            module_id = %spec.module_id,
3326                            program = %spec.program.display(),
3327                            error = %err,
3328                            "configured module failed to spawn; marking failed and continuing"
3329                        );
3330                    }
3331                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
3332                        state.state = ModuleState::Failed;
3333                        clear_current_process_facts(state);
3334                    })?;
3335                    None
3336                }
3337            }
3338        } else {
3339            None
3340        };
3341        self.start_monitor(spec, runtime, &module, child, commands);
3342        Ok(module)
3343    }
3344
3345    fn runtime_config(&self) -> SupervisorRuntimeConfig {
3346        let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3347        SupervisorRuntimeConfig {
3348            start_requester: None,
3349            pending_activity_restart: false,
3350            provenance_probe: self.provenance_probe.clone(),
3351            scheduled_respawn: Arc::default(),
3352            deferred_reload_reply: Arc::default(),
3353            restart_policy: self.restart_policy,
3354            drain_timeout: self.drain_timeout,
3355            // Shared with this module's roster copy: daemon shutdown waits on
3356            // each child for the module's own drain budget, as resolved now.
3357            child_roster: self
3358                .child_roster
3359                .for_module(Arc::clone(&effective_drain_timeout)),
3360            effective_drain_timeout,
3361            default_drain_timeout: self.drain_timeout,
3362            health: self.health.clone(),
3363            connection_file_path: self.connection_file_path.clone(),
3364            capture_logs_dir: self.capture_logs_dir.clone(),
3365            forwarding: self.forwarding.clone(),
3366            supervisor_handle: self.supervisor_handle.clone(),
3367            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3368            terminal_ring: Arc::new(Mutex::new(
3369                TerminalRing::new(
3370                    TerminalRingConfig::default(),
3371                    self.daemon_start_clock.started_at_ms(),
3372                )
3373                .with_start_clock(self.daemon_start_clock)
3374                .with_journal(self.terminal_journal.clone())
3375                .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3376            )),
3377            spawn_events: self.spawn_events.clone(),
3378            #[cfg(target_os = "linux")]
3379            cgroup_placement: self.cgroup_placement.clone(),
3380            #[cfg(test)]
3381            test_seed_stale_facts_before_enable_spawn: false,
3382            #[cfg(test)]
3383            test_reload_exit_record_gate: None,
3384        }
3385    }
3386
3387    fn supervised_module(
3388        &self,
3389        spec: ModuleSpec,
3390        runtime: SupervisorRuntimeConfig,
3391        snapshot: SharedSnapshot,
3392        child: Option<SupervisedChild>,
3393    ) -> SupervisedModule {
3394        let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3395        self.start_monitor(spec, runtime, &module, child, commands);
3396        module
3397    }
3398
3399    fn register_module(
3400        &self,
3401        spec: &ModuleSpec,
3402        runtime: &SupervisorRuntimeConfig,
3403        snapshot: SharedSnapshot,
3404    ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3405        let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3406            spec: spec.clone(),
3407            health: runtime.health.clone(),
3408        }));
3409        let stderr_ring = Arc::clone(&runtime.stderr_ring);
3410        let terminal_ring = Arc::clone(&runtime.terminal_ring);
3411        // The module's OWN policy, which may be its per-module config rather than
3412        // the supervisor-wide one; status must report the budget the supervise
3413        // loop actually enforces.
3414        let restart_policy = runtime.restart_policy;
3415        let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3416        let (tx, rx) = mpsc::channel(4);
3417        let module_id = spec.module_id.clone();
3418        let module = SupervisedModule {
3419            inner: Arc::new(SupervisedModuleInner {
3420                module_id: module_id.clone(),
3421                registry: Arc::clone(&self.registry),
3422                snapshot,
3423                configuration,
3424                stderr_ring,
3425                terminal_ring,
3426                commands: tx,
3427                monitor: Mutex::new(None),
3428                restart_policy,
3429                effective_drain_timeout,
3430                provenance_probe: self.provenance_probe.clone(),
3431            }),
3432        };
3433        // Identity gates and the configured mark must precede the roster entry,
3434        // so a client never sees a module without its admission policy.
3435        if let Some(supervisor_handle) = &self.supervisor_handle {
3436            supervisor_handle.insert(module.clone());
3437        }
3438        (module, rx)
3439    }
3440
3441    fn start_monitor(
3442        &self,
3443        spec: ModuleSpec,
3444        runtime: SupervisorRuntimeConfig,
3445        module: &SupervisedModule,
3446        child: Option<SupervisedChild>,
3447        commands: mpsc::Receiver<SupervisorCommand>,
3448    ) {
3449        let monitor = tokio::spawn(supervise_loop(
3450            spec,
3451            runtime,
3452            Arc::clone(&self.registry),
3453            Arc::clone(&self.process_liveness),
3454            Arc::clone(&module.inner.snapshot),
3455            child,
3456            commands,
3457        ));
3458        *module
3459            .inner
3460            .monitor
3461            .lock()
3462            .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3463    }
3464}
3465
3466impl Default for Supervisor {
3467    fn default() -> Self {
3468        Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3469    }
3470}
3471
3472/// Handle to one supervised child process.
3473#[derive(Clone)]
3474pub struct SupervisedModule {
3475    inner: Arc<SupervisedModuleInner>,
3476}
3477
3478struct SupervisedModuleInner {
3479    module_id: String,
3480    registry: Arc<Registry>,
3481    snapshot: SharedSnapshot,
3482    configuration: Arc<Mutex<SupervisedConfiguration>>,
3483    stderr_ring: Arc<Mutex<StderrRing>>,
3484    terminal_ring: Arc<Mutex<TerminalRing>>,
3485    commands: mpsc::Sender<SupervisorCommand>,
3486    monitor: Mutex<Option<JoinHandle<()>>>,
3487    /// Copied from the supervisor's runtime config at spawn so `status()` can
3488    /// report the restart budget without reaching back into the supervisor. The
3489    /// policy is fixed for the process's lifetime, so a copy cannot drift.
3490    restart_policy: RestartPolicy,
3491    effective_drain_timeout: Arc<Mutex<Duration>>,
3492    provenance_probe: ExecutableIdentityProbe,
3493}
3494
3495impl fmt::Debug for SupervisedModule {
3496    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3497        f.debug_struct("SupervisedModule")
3498            .field("module_id", &self.inner.module_id)
3499            .field("status", &self.status())
3500            .finish_non_exhaustive()
3501    }
3502}
3503
3504impl SupervisedModule {
3505    pub fn module_id(&self) -> &str {
3506        &self.inner.module_id
3507    }
3508
3509    /// Test-only: put one probe miss on the streak, the way
3510    /// `handle_health_probe_failure` does, so tests can assert what a later
3511    /// event does to the streak without driving the whole probe loop.
3512    #[cfg(test)]
3513    pub(crate) fn record_health_probe_failure_for_test(
3514        &self,
3515        detail: &str,
3516    ) -> Result<(), SuperviseError> {
3517        update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3518            state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3519            state.health.detail = Some(detail.to_string());
3520        })
3521    }
3522
3523    pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3524        Ok(lock_snapshot(&self.inner.snapshot)?.state)
3525    }
3526
3527    pub(crate) fn registration_end_reason(
3528        &self,
3529    ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3530        let snapshot = lock_snapshot(&self.inner.snapshot)?;
3531        Ok(match snapshot.state {
3532            ModuleState::Draining if snapshot.draining_to_replace => {
3533                Some(RegistrationEndReason::SupervisorRestart)
3534            }
3535            ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3536            ModuleState::Restarting if snapshot.draining_to_replace => {
3537                Some(RegistrationEndReason::SupervisorRestart)
3538            }
3539            _ => None,
3540        })
3541    }
3542
3543    /// The module's retained stderr, newest lines last.
3544    ///
3545    /// Deliberately NOT on [`Self::status`]: a bounded tail is kilobytes per
3546    /// module, `supervisor.list` renders every module, and putting it in the
3547    /// shared snapshot would make each status read carry a payload almost nobody
3548    /// asked for. Callers that want the text ask for it.
3549    pub fn stderr_tail(
3550        &self,
3551        max_lines: Option<usize>,
3552        max_bytes: Option<usize>,
3553    ) -> StderrTailSnapshot {
3554        self.inner
3555            .stderr_ring
3556            .lock()
3557            .unwrap_or_else(|poisoned| poisoned.into_inner())
3558            .snapshot(max_lines, max_bytes)
3559    }
3560
3561    /// The module's bounded terminal history, oldest retained exit first.
3562    ///
3563    /// The daemon-start stamp distinguishes a quiet supervisor from a replacement
3564    /// daemon whose in-memory history was necessarily reset.
3565    pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3566        self.inner
3567            .terminal_ring
3568            .lock()
3569            .unwrap_or_else(|poisoned| poisoned.into_inner())
3570            .snapshot()
3571    }
3572
3573    /// Retained observations from the current ring and all journal generations.
3574    ///
3575    /// Blocking: this reads the journal files. Async callers use
3576    /// [`Self::read_durable_terminal_history`].
3577    pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3578        durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3579    }
3580
3581    /// [`Self::durable_terminal_history`] on a blocking thread, so the journal
3582    /// read (up to every retained generation) never occupies a runtime worker.
3583    /// Fails only if the blocking task could not finish (runtime shutdown or a
3584    /// panic in the read).
3585    pub(crate) async fn read_durable_terminal_history(
3586        &self,
3587    ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3588        let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3589        let module_id = self.inner.module_id.clone();
3590        tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3591            .await
3592    }
3593
3594    pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3595        self.status_with_snapshot_lock(&self.inner.snapshot, None)
3596            .map(|(status, _)| status)
3597    }
3598
3599    pub(crate) fn record_deliberate_severance(
3600        &self,
3601        identity: ProcessIdentity,
3602    ) -> Result<bool, SuperviseError> {
3603        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3604        if snapshot.pid != Some(identity.pid)
3605            || snapshot.process_start_time != Some(identity.start_time)
3606        {
3607            return Ok(false);
3608        }
3609        snapshot.deliberate_severance = Some(identity);
3610        Ok(true)
3611    }
3612
3613    /// Read status for a channel-0 renderer and report a contended snapshot lock.
3614    ///
3615    /// Internal supervision callers use [`Self::status`] so writer-side machinery
3616    /// does not produce reader-observability logs.
3617    pub(crate) fn status_for_control(
3618        &self,
3619        caller: &'static str,
3620    ) -> Result<ModuleStatus, SuperviseError> {
3621        self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3622            .map(|(status, _)| status)
3623    }
3624
3625    fn status_with_snapshot_lock(
3626        &self,
3627        snapshot: &SharedSnapshot,
3628        caller: Option<&'static str>,
3629    ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3630        let mut guard = match caller {
3631            Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3632            None => lock_snapshot(snapshot)?,
3633        };
3634        // Read the budget through the pruning path so a reader sees the same
3635        // in-window count the restart decision would use, not a stale total.
3636        let restart_count =
3637            guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3638        let snapshot = guard.clone();
3639        drop(guard);
3640        let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3641            SuperviseError::StatePoisoned {
3642                module_id: Some(self.inner.module_id.clone()),
3643            }
3644        })?;
3645        let registration_active = self
3646            .inner
3647            .registry
3648            .get_module(&self.inner.module_id)
3649            .map_err(SuperviseError::Registry)?
3650            .is_some();
3651        let protocol = snapshot
3652            .spawned_protocol
3653            .unwrap_or(self.declared_protocol()?);
3654        let running_process =
3655            snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3656        // Registration is the difference between the two protocols and the only
3657        // one: a subc module that has not registered cannot serve a request even
3658        // though its process is up, and a `none` module never registers at all,
3659        // so requiring it there would pin `live` to false for the whole life of
3660        // a perfectly healthy process.
3661        let live = match protocol {
3662            ModuleProtocol::Subc => running_process && registration_active,
3663            ModuleProtocol::None => running_process,
3664        };
3665
3666        Ok((
3667            ModuleStatus {
3668                waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3669                waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3670                module_id: self.inner.module_id.clone(),
3671                launch_nonce_source: observed_launch_nonce_source(&snapshot),
3672                state: snapshot.state,
3673                enabled: snapshot.enabled,
3674                process_alive: snapshot.process_alive,
3675                registration_active,
3676                protocol,
3677                live,
3678                restart_count,
3679                lifetime_restarts: snapshot.lifetime_restarts,
3680                spawn_generation: snapshot.spawn_generation,
3681                max_restarts: self.inner.restart_policy.max_restarts,
3682                restart_window: self.inner.restart_policy.window,
3683                drain_timeout,
3684                restart_backoff: self.inner.restart_policy.backoff,
3685                restart_max_backoff: self.inner.restart_policy.max_backoff,
3686                pid: snapshot.reported_pid(),
3687                spawned_at_ms: snapshot.spawned_at_ms,
3688                spawned_from: snapshot.spawned_from,
3689                launch_spec_digest: snapshot.launch_spec_digest,
3690                process_start_time: snapshot.process_start_time,
3691                last_exit: snapshot.last_exit,
3692                health: snapshot.health,
3693            },
3694            snapshot.spawned_file_identity.clone(),
3695        ))
3696    }
3697
3698    #[cfg(test)]
3699    pub(crate) fn hold_snapshot_for_test(
3700        &self,
3701        acquired: std::sync::mpsc::Sender<()>,
3702        hold: Duration,
3703    ) -> std::thread::JoinHandle<()> {
3704        let snapshot = Arc::clone(&self.inner.snapshot);
3705        std::thread::spawn(move || {
3706            let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3707            acquired
3708                .send(())
3709                .expect("test receiver waits for snapshot lock");
3710            std::thread::sleep(hold);
3711        })
3712    }
3713
3714    /// The status and the running-image check for `supervisor.provenance`,
3715    /// taken from one status read. The exec acknowledgement can land between
3716    /// two separate reads, and the reply would then pair "no pid yet" with an
3717    /// image observed after the module started, which describes no single
3718    /// moment.
3719    pub(crate) async fn status_and_running_image_agreement(
3720        &self,
3721    ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3722        let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3723        let image = self
3724            .inner
3725            .provenance_probe
3726            .observe(
3727                status.pid,
3728                status.spawned_from.as_deref(),
3729                identity,
3730                status.process_start_time,
3731            )
3732            .await;
3733        Ok((status, image))
3734    }
3735
3736    pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3737        let snapshot = match lock_snapshot(&self.inner.snapshot) {
3738            Ok(snapshot) => snapshot.clone(),
3739            Err(_) => {
3740                return subc_control::RunningImageAgreement::Unavailable {
3741                    reason: subc_control::RunningImageUnavailableReason::NotRunning,
3742                };
3743            }
3744        };
3745        self.inner
3746            .provenance_probe
3747            .observe(
3748                snapshot.reported_pid(),
3749                snapshot.spawned_from.as_deref(),
3750                snapshot.spawned_file_identity.clone(),
3751                snapshot.process_start_time,
3752            )
3753            .await
3754    }
3755
3756    /// Memory and CPU time of the module's current process, read now. Only the
3757    /// process the supervisor spawned is read, not processes it has started.
3758    pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3759        #[cfg(windows)]
3760        {
3761            let image = lock_snapshot(&self.inner.snapshot)
3762                .ok()
3763                .and_then(|snapshot| {
3764                    snapshot
3765                        .spawned_file_identity
3766                        .as_ref()
3767                        .and_then(|identity| identity.windows_image.clone())
3768                });
3769            match image {
3770                Some(image) => {
3771                    crate::child_resources::read_windows_handle(image.process(), image.start_time())
3772                }
3773                None => subc_control::ChildResourceUsage::Unavailable {
3774                    reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3775                },
3776            }
3777        }
3778        #[cfg(not(windows))]
3779        {
3780            let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3781                Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3782                Err(_) => {
3783                    return subc_control::ChildResourceUsage::Unavailable {
3784                        reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3785                    }
3786                }
3787            };
3788            crate::child_resources::read(pid, start_time)
3789        }
3790    }
3791
3792    pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3793        let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3794        Ok(match snapshot.state {
3795            ModuleState::Restarting => true,
3796            ModuleState::Failed | ModuleState::Disabled => false,
3797            _ if snapshot.enabled && snapshot.respawn_pending => true,
3798            _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3799        })
3800    }
3801
3802    #[cfg(test)]
3803    pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3804        self.is_warming_with_snapshot_lock(None)
3805    }
3806
3807    pub(crate) fn is_warming_for_control(
3808        &self,
3809        caller: &'static str,
3810    ) -> Result<bool, SuperviseError> {
3811        self.is_warming_with_snapshot_lock(Some(caller))
3812    }
3813
3814    fn is_warming_with_snapshot_lock(
3815        &self,
3816        caller: Option<&'static str>,
3817    ) -> Result<bool, SuperviseError> {
3818        let snapshot = match caller {
3819            Some(caller) => {
3820                lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3821            }
3822            None => lock_snapshot(&self.inner.snapshot)?,
3823        }
3824        .clone();
3825        Ok(matches!(
3826            snapshot.state,
3827            ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3828        ))
3829    }
3830
3831    /// Drain the module and stop monitoring it.
3832    pub async fn drain(&self) -> Result<(), SuperviseError> {
3833        self.stop().await
3834    }
3835
3836    pub(crate) async fn retire_requested(
3837        &self,
3838        requester: Option<Principal>,
3839    ) -> Result<(), SuperviseError> {
3840        match self.state()? {
3841            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3842            ModuleState::Starting
3843            | ModuleState::Running
3844            | ModuleState::Unresponsive
3845            | ModuleState::Restarting
3846            | ModuleState::Draining
3847            | ModuleState::Disabled => {}
3848        }
3849
3850        let (reply_tx, reply_rx) = oneshot::channel();
3851        self.inner
3852            .commands
3853            .send(SupervisorCommand::Retire {
3854                requester,
3855                reply: reply_tx,
3856            })
3857            .await
3858            .map_err(|_| SuperviseError::CommandClosed {
3859                module_id: self.inner.module_id.clone(),
3860            })?;
3861        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3862            module_id: self.inner.module_id.clone(),
3863        })?
3864    }
3865
3866    pub async fn stop(&self) -> Result<(), SuperviseError> {
3867        self.stop_requested(None).await
3868    }
3869
3870    pub(crate) async fn stop_requested(
3871        &self,
3872        requester: Option<Principal>,
3873    ) -> Result<(), SuperviseError> {
3874        match self.state()? {
3875            ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3876            ModuleState::Starting
3877            | ModuleState::Running
3878            | ModuleState::Unresponsive
3879            | ModuleState::Restarting
3880            | ModuleState::Draining
3881            | ModuleState::Disabled => {}
3882        }
3883
3884        let (reply_tx, reply_rx) = oneshot::channel();
3885        self.inner
3886            .commands
3887            .send(SupervisorCommand::Drain {
3888                requester,
3889                reply: reply_tx,
3890            })
3891            .await
3892            .map_err(|_| SuperviseError::CommandClosed {
3893                module_id: self.inner.module_id.clone(),
3894            })?;
3895        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3896            module_id: self.inner.module_id.clone(),
3897        })?
3898    }
3899
3900    pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3901        self.restart_requested(drain_timeout_ms, None).await
3902    }
3903
3904    pub(crate) async fn restart_requested(
3905        &self,
3906        drain_timeout_ms: Option<u64>,
3907        requester: Option<Principal>,
3908    ) -> Result<(), SuperviseError> {
3909        let queued_at = Instant::now();
3910        let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3911        let (reply_tx, reply_rx) = oneshot::channel();
3912        self.inner
3913            .commands
3914            .send(SupervisorCommand::Restart {
3915                requester,
3916                drain_timeout_ms,
3917                received_at_generation,
3918                queued_at,
3919                reply: reply_tx,
3920            })
3921            .await
3922            .map_err(|_| SuperviseError::CommandClosed {
3923                module_id: self.inner.module_id.clone(),
3924            })?;
3925        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3926            module_id: self.inner.module_id.clone(),
3927        })?
3928    }
3929
3930    /// Blue/green restart: see [`SupervisorCommand::Swap`] and the
3931    /// `supervisor_swap` module. Returns once the swap has cut over (the old
3932    /// process then drains in the background of the supervise loop) or has
3933    /// failed, leaving the old process serving.
3934    pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3935        self.swap_requested(ready_timeout, None).await
3936    }
3937
3938    pub(crate) async fn swap_requested(
3939        &self,
3940        ready_timeout: Option<Duration>,
3941        requester: Option<Principal>,
3942    ) -> Result<(), SuperviseError> {
3943        let (reply_tx, reply_rx) = oneshot::channel();
3944        self.inner
3945            .commands
3946            .send(SupervisorCommand::Swap {
3947                requester,
3948                ready_timeout,
3949                reply: reply_tx,
3950            })
3951            .await
3952            .map_err(|_| SuperviseError::CommandClosed {
3953                module_id: self.inner.module_id.clone(),
3954            })?;
3955        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3956            module_id: self.inner.module_id.clone(),
3957        })?
3958    }
3959
3960    pub async fn reload(&self) -> Result<(), SuperviseError> {
3961        self.reload_requested(None).await
3962    }
3963
3964    pub(crate) async fn reload_requested(
3965        &self,
3966        requester: Option<Principal>,
3967    ) -> Result<(), SuperviseError> {
3968        let (reply_tx, reply_rx) = oneshot::channel();
3969        self.inner
3970            .commands
3971            .send(SupervisorCommand::Reload {
3972                requester,
3973                reply: reply_tx,
3974            })
3975            .await
3976            .map_err(|_| SuperviseError::CommandClosed {
3977                module_id: self.inner.module_id.clone(),
3978            })?;
3979        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3980            module_id: self.inner.module_id.clone(),
3981        })?
3982    }
3983
3984    pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3985        self.set_enabled_requested(enabled, None).await
3986    }
3987
3988    pub(crate) async fn set_enabled_requested(
3989        &self,
3990        enabled: bool,
3991        requester: Option<Principal>,
3992    ) -> Result<bool, SuperviseError> {
3993        let (reply_tx, reply_rx) = oneshot::channel();
3994        self.inner
3995            .commands
3996            .send(SupervisorCommand::SetEnabled {
3997                requester,
3998                enabled,
3999                reply: reply_tx,
4000            })
4001            .await
4002            .map_err(|_| SuperviseError::CommandClosed {
4003                module_id: self.inner.module_id.clone(),
4004            })?;
4005        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4006            module_id: self.inner.module_id.clone(),
4007        })?
4008    }
4009
4010    /// The current process's protocol, or the configured protocol when down.
4011    /// A rescan stores the next launch spec without changing how an existing
4012    /// process registers, serves routes, is probed, or exits.
4013    pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4014        let configured = self
4015            .inner
4016            .configuration
4017            .lock()
4018            .map_err(|_| SuperviseError::StatePoisoned {
4019                module_id: Some(self.inner.module_id.clone()),
4020            })?
4021            .spec
4022            .protocol;
4023        let state = lock_snapshot(&self.inner.snapshot)?;
4024        Ok(state.spawned_protocol.unwrap_or(configured))
4025    }
4026
4027    pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4028        let configuration =
4029            self.inner
4030                .configuration
4031                .lock()
4032                .map_err(|_| SuperviseError::StatePoisoned {
4033                    module_id: Some(self.inner.module_id.clone()),
4034                })?;
4035        Ok((configuration.spec.clone(), configuration.health.clone()))
4036    }
4037
4038    /// Replace this module's launch spec, keeping its health and drain policy,
4039    /// the way a rescan does for a changed config entry. The running process is
4040    /// untouched; the next spawn (a restart, or a swap's candidate) uses it.
4041    #[cfg(any(test, feature = "test-support"))]
4042    pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4043        let (_, health) = self.configuration()?;
4044        let drain_timeout_ms = u64::try_from(
4045            self.inner
4046                .effective_drain_timeout
4047                .lock()
4048                .unwrap_or_else(|poisoned| poisoned.into_inner())
4049                .as_millis(),
4050        )
4051        .ok();
4052        self.update_configuration(spec, health, drain_timeout_ms)
4053            .await
4054    }
4055
4056    pub(crate) async fn update_configuration(
4057        &self,
4058        spec: ModuleSpec,
4059        health: HealthConfig,
4060        drain_timeout_ms: Option<u64>,
4061    ) -> Result<(), SuperviseError> {
4062        if spec.module_id != self.inner.module_id {
4063            return Err(SuperviseError::InvalidSpec {
4064                reason: "a supervised module's module_id cannot be changed".to_string(),
4065            });
4066        }
4067        validate_spec(&spec)?;
4068        let (reply_tx, reply_rx) = oneshot::channel();
4069        self.inner
4070            .commands
4071            .send(SupervisorCommand::UpdateConfiguration {
4072                spec: spec.clone(),
4073                health: health.clone(),
4074                drain_timeout_ms,
4075                reply: reply_tx,
4076            })
4077            .await
4078            .map_err(|_| SuperviseError::CommandClosed {
4079                module_id: self.inner.module_id.clone(),
4080            })?;
4081        reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4082            module_id: self.inner.module_id.clone(),
4083        })?;
4084        let mut configuration =
4085            self.inner
4086                .configuration
4087                .lock()
4088                .map_err(|_| SuperviseError::StatePoisoned {
4089                    module_id: Some(self.inner.module_id.clone()),
4090                })?;
4091        configuration.spec = spec;
4092        configuration.health = health;
4093        Ok(())
4094    }
4095}
4096
4097impl Drop for SupervisedModuleInner {
4098    fn drop(&mut self) {
4099        let Ok(mut monitor) = self.monitor.lock() else {
4100            return;
4101        };
4102        if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4103            let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4104                state.state = ModuleState::Stopped;
4105                clear_current_process_facts(state);
4106            });
4107            monitor.abort();
4108        }
4109        let _ = monitor.take();
4110    }
4111}
4112
4113#[derive(Debug)]
4114enum SupervisorCommand {
4115    Drain {
4116        requester: Option<Principal>,
4117        reply: oneshot::Sender<Result<(), SuperviseError>>,
4118    },
4119    Retire {
4120        requester: Option<Principal>,
4121        reply: oneshot::Sender<Result<(), SuperviseError>>,
4122    },
4123    Restart {
4124        requester: Option<Principal>,
4125        /// Operator override for this one restart's drain budget, in ms. `None`
4126        /// uses the module's configured/default budget; `Some(0)` cuts
4127        /// immediately (wedge bounce: a stuck request never settles, so
4128        /// waiting only delays recovery).
4129        drain_timeout_ms: Option<u64>,
4130        /// The module's `spawn_generation` when the request was received, before
4131        /// it waited in the command queue. A queued restart whose module has
4132        /// since spawned a newer process is already satisfied (see the handler).
4133        received_at_generation: u64,
4134        /// When the request entered the command queue, so the handler can log
4135        /// how long it waited behind the loop's other work.
4136        queued_at: Instant,
4137        reply: oneshot::Sender<Result<(), SuperviseError>>,
4138    },
4139    Reload {
4140        requester: Option<Principal>,
4141        reply: oneshot::Sender<Result<(), SuperviseError>>,
4142    },
4143    SetEnabled {
4144        requester: Option<Principal>,
4145        enabled: bool,
4146        reply: oneshot::Sender<Result<bool, SuperviseError>>,
4147    },
4148    UpdateConfiguration {
4149        spec: ModuleSpec,
4150        health: HealthConfig,
4151        /// Per-module drain override from the new config; `None` re-resolves to
4152        /// the supervisor-wide default.
4153        drain_timeout_ms: Option<u64>,
4154        reply: oneshot::Sender<()>,
4155    },
4156    Swap {
4157        requester: Option<Principal>,
4158        /// How long the candidate may take to register and declare itself
4159        /// ready. `None` uses [`DEFAULT_SWAP_READY_TIMEOUT`].
4160        ready_timeout: Option<Duration>,
4161        /// Answered at cutover or failure; the incumbent's drain follows.
4162        reply: oneshot::Sender<Result<(), SuperviseError>>,
4163    },
4164}
4165
4166#[derive(Debug)]
4167pub enum SuperviseError {
4168    InvalidSpec {
4169        reason: String,
4170    },
4171    Spawn {
4172        program: PathBuf,
4173        source: io::Error,
4174        cgroup_path: Option<PathBuf>,
4175    },
4176    Cgroup {
4177        module_id: String,
4178        source: io::Error,
4179    },
4180    /// CSPRNG failure generating a reserved module's launch nonce. Fail loud rather
4181    /// than spawn a reserved module without its identity binding.
4182    LaunchNonce {
4183        reason: String,
4184    },
4185    Wait {
4186        module_id: String,
4187        source: io::Error,
4188    },
4189    Kill {
4190        module_id: String,
4191        source: io::Error,
4192    },
4193    Forwarding(ForwardingError),
4194    Registry(RegistryError),
4195    ReloadUnavailable {
4196        module_id: String,
4197        reason: String,
4198    },
4199    /// An operator restart/reload was requested for a module that is currently
4200    /// disabled. Restart/reload cycle a *running* module; a disabled module must
4201    /// be explicitly re-enabled (set_enabled(true)) rather than silently started
4202    /// by a restart, so these commands are rejected instead of re-enabling it.
4203    Disabled {
4204        module_id: String,
4205    },
4206    ReloadFailed {
4207        module_id: String,
4208        reason: String,
4209    },
4210    RegistrationStillActive {
4211        module_id: String,
4212        waited: Duration,
4213    },
4214    StatePoisoned {
4215        module_id: Option<String>,
4216    },
4217    CommandClosed {
4218        module_id: String,
4219    },
4220    /// A restart or reload arrived while a swap's candidate was warming. The
4221    /// swap owns the module until it cuts over or fails; a stop or disable
4222    /// would have aborted it instead.
4223    SwapInProgress {
4224        module_id: String,
4225    },
4226    /// A swap was refused before anything was spawned.
4227    SwapRefused {
4228        module_id: String,
4229        reason: SwapRefusal,
4230    },
4231    /// A swap spawned a candidate and gave up on it. The candidate has been
4232    /// killed and its slot freed; the incumbent was left serving and was never
4233    /// drained, except in the one `CutoverLost` case described on that arm.
4234    SwapFailed {
4235        module_id: String,
4236        arm: SwapFailureArm,
4237        detail: String,
4238        /// How the candidate exited, when it exited on its own before the
4239        /// supervisor gave up on it.
4240        candidate_exit: Option<ExitReport>,
4241    },
4242}
4243
4244/// Why a swap was refused before a candidate was spawned.
4245#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4246pub enum SwapRefusal {
4247    /// The module's config does not declare `overlap: "safe"`.
4248    OverlapExclusive,
4249    /// The module is not registered, so there is no incumbent to keep serving
4250    /// and nothing a swap would improve on; a plain restart is the tool.
4251    NotRegistered,
4252    /// The module does not speak the subc wire, so a candidate could never
4253    /// register or declare itself ready.
4254    ProtocolNone,
4255    /// The supervisor lacks the forwarding table (to cut routes over) or the
4256    /// shared handle (to admit the candidate's HELLO) that a swap needs.
4257    NotConfigured,
4258    /// A swap is already open for this module.
4259    AlreadySwapping,
4260}
4261
4262impl SwapRefusal {
4263    pub fn as_str(self) -> &'static str {
4264        match self {
4265            Self::OverlapExclusive => "overlap_exclusive",
4266            Self::NotRegistered => "not_registered",
4267            Self::ProtocolNone => "protocol_none",
4268            Self::NotConfigured => "not_configured",
4269            Self::AlreadySwapping => "already_swapping",
4270        }
4271    }
4272}
4273
4274/// Which failure arm ended a swap. Every arm but one leaves the incumbent
4275/// serving and undrained; see `CutoverLost`.
4276#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4277pub enum SwapFailureArm {
4278    /// The candidate process could not be started.
4279    SpawnFailed,
4280    /// The candidate did not register within the readiness budget.
4281    NeverRegistered,
4282    /// The candidate registered but did not declare itself ready in time.
4283    NeverReady,
4284    /// The candidate exited before cutover.
4285    CandidateExited,
4286    /// The candidate declared itself ready but failed its health probe.
4287    CandidateUnhealthy,
4288    /// An operator stop, disable or retire arrived while the candidate warmed.
4289    /// The candidate was killed and the operator's command then carried out on
4290    /// the incumbent.
4291    Interrupted,
4292    /// The candidate's connection closed at the moment of cutover. If it
4293    /// closed before forwarding moved, the incumbent is untouched. If it closed
4294    /// between the forwarding and registry halves of cutover, forwarding can no
4295    /// longer route to the incumbent, so the module is restarted plainly.
4296    CutoverLost,
4297}
4298
4299impl SwapFailureArm {
4300    pub fn as_str(self) -> &'static str {
4301        match self {
4302            Self::SpawnFailed => "spawn_failed",
4303            Self::NeverRegistered => "never_registered",
4304            Self::NeverReady => "never_ready",
4305            Self::CandidateExited => "candidate_exited",
4306            Self::CandidateUnhealthy => "candidate_unhealthy",
4307            Self::Interrupted => "interrupted",
4308            Self::CutoverLost => "cutover_lost",
4309        }
4310    }
4311}
4312
4313impl fmt::Display for SuperviseError {
4314    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4315        match self {
4316            Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4317            Self::Spawn {
4318                program,
4319                source,
4320                cgroup_path: Some(cgroup_path),
4321            } => write!(
4322                f,
4323                "failed to place module in cgroup '{}' while spawning '{}': {source}",
4324                cgroup_path.display(),
4325                program.display()
4326            ),
4327            Self::Spawn {
4328                program,
4329                source,
4330                cgroup_path: None,
4331            } => write!(
4332                f,
4333                "failed to spawn module '{}': {source}",
4334                program.display()
4335            ),
4336            Self::Cgroup { module_id, source } => {
4337                write!(
4338                    f,
4339                    "failed to prepare cgroup for module '{module_id}': {source}"
4340                )
4341            }
4342            Self::LaunchNonce { reason } => {
4343                write!(
4344                    f,
4345                    "failed to generate reserved-module launch nonce: {reason}"
4346                )
4347            }
4348            Self::Wait { module_id, source } => {
4349                write!(f, "failed to wait for module '{module_id}': {source}")
4350            }
4351            Self::Kill { module_id, source } => {
4352                write!(f, "failed to kill module '{module_id}': {source}")
4353            }
4354            Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4355            Self::Registry(err) => write!(f, "registry error: {err}"),
4356            Self::ReloadUnavailable { module_id, reason } => {
4357                write!(f, "reload unavailable for module '{module_id}': {reason}")
4358            }
4359            Self::Disabled { module_id } => {
4360                write!(
4361                    f,
4362                    "module '{module_id}' is disabled; enable it before restart or reload"
4363                )
4364            }
4365            Self::ReloadFailed { module_id, reason } => {
4366                write!(f, "reload failed for module '{module_id}': {reason}")
4367            }
4368            Self::RegistrationStillActive { module_id, waited } => write!(
4369                f,
4370                "module '{module_id}' registration remained active after waiting {waited:?}"
4371            ),
4372            Self::StatePoisoned { module_id } => match module_id {
4373                Some(module_id) => {
4374                    write!(f, "supervisor state for module '{module_id}' was poisoned")
4375                }
4376                None => write!(f, "supervisor state was poisoned"),
4377            },
4378            Self::CommandClosed { module_id } => {
4379                write!(
4380                    f,
4381                    "supervisor command channel for module '{module_id}' is closed"
4382                )
4383            }
4384            Self::SwapInProgress { module_id } => write!(
4385                f,
4386                "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4387            ),
4388            Self::SwapRefused { module_id, reason } => match reason {
4389                SwapRefusal::OverlapExclusive => write!(
4390                    f,
4391                    "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4392                ),
4393                SwapRefusal::NotRegistered => write!(
4394                    f,
4395                    "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4396                ),
4397                SwapRefusal::ProtocolNone => write!(
4398                    f,
4399                    "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4400                ),
4401                SwapRefusal::NotConfigured => write!(
4402                    f,
4403                    "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4404                ),
4405                SwapRefusal::AlreadySwapping => {
4406                    write!(f, "module '{module_id}' is already being swapped")
4407                }
4408            },
4409            Self::SwapFailed {
4410                module_id,
4411                arm,
4412                detail,
4413                ..
4414            } => write!(
4415                f,
4416                "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4417                arm.as_str()
4418            ),
4419        }
4420    }
4421}
4422
4423impl Error for SuperviseError {
4424    fn source(&self) -> Option<&(dyn Error + 'static)> {
4425        match self {
4426            Self::Spawn { source, .. }
4427            | Self::Cgroup { source, .. }
4428            | Self::Wait { source, .. }
4429            | Self::Kill { source, .. } => Some(source),
4430            Self::Forwarding(err) => Some(err),
4431            Self::Registry(err) => Some(err),
4432            Self::LaunchNonce { .. }
4433            | Self::InvalidSpec { .. }
4434            | Self::ReloadUnavailable { .. }
4435            | Self::Disabled { .. }
4436            | Self::ReloadFailed { .. }
4437            | Self::RegistrationStillActive { .. }
4438            | Self::StatePoisoned { .. }
4439            | Self::CommandClosed { .. }
4440            | Self::SwapInProgress { .. }
4441            | Self::SwapRefused { .. }
4442            | Self::SwapFailed { .. } => None,
4443        }
4444    }
4445}
4446
4447pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4448    if spec.module_id.trim().is_empty() {
4449        return Err(SuperviseError::InvalidSpec {
4450            reason: "module_id must not be empty".to_string(),
4451        });
4452    }
4453
4454    Ok(())
4455}
4456
4457#[derive(Debug, Default)]
4458struct HealthProbeRuntime {
4459    spawn_generation: Option<u64>,
4460    configured_health: Option<HealthConfig>,
4461    registered_connection: Option<crate::ConnectionId>,
4462    advertised: bool,
4463    next_probe_at: Option<Instant>,
4464    probe_index: u64,
4465}
4466
4467fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4468    lock_snapshot(snapshot)
4469        .ok()
4470        .and_then(|state| state.spawned_protocol)
4471        .unwrap_or(spec.protocol)
4472}
4473
4474impl HealthProbeRuntime {
4475    fn refresh_registration(
4476        &mut self,
4477        spec: &ModuleSpec,
4478        runtime: &SupervisorRuntimeConfig,
4479        registry: &Registry,
4480        snapshot: &SharedSnapshot,
4481    ) {
4482        let generation = lock_snapshot(snapshot)
4483            .ok()
4484            .map(|state| state.spawn_generation);
4485        if self.spawn_generation != generation {
4486            // A replacement cannot reuse a deadline that is already due for its
4487            // predecessor. HTTP children get a full cadence to start; wire
4488            // children still wait for their own health-advertising HELLO.
4489            *self = Self {
4490                spawn_generation: generation,
4491                ..Self::default()
4492            };
4493        }
4494        if self.configured_health.as_ref() != Some(&runtime.health) {
4495            self.configured_health = Some(runtime.health.clone());
4496            self.next_probe_at = None;
4497            self.registered_connection = None;
4498            self.probe_index = 0;
4499        }
4500        // A non-wire process never registers. Only an explicitly configured
4501        // HTTP endpoint can arm its health probe; an absent HELLO is not a
4502        // health failure for that kind of process.
4503        if running_protocol(spec, snapshot) == ModuleProtocol::None {
4504            self.registered_connection = None;
4505            self.advertised = runtime.health.http.is_some();
4506            if !self.advertised {
4507                self.next_probe_at = None;
4508                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4509                    let unknown = ModuleHealthStatus::default();
4510                    if state.health != unknown {
4511                        state.health = unknown;
4512                    }
4513                });
4514            } else if self.next_probe_at.is_none() {
4515                self.next_probe_at = Some(
4516                    Instant::now()
4517                        + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4518                );
4519            }
4520            return;
4521        }
4522
4523        let registration = match registry.get_module(&spec.module_id) {
4524            Ok(registration) => registration,
4525            Err(err) => {
4526                warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4527                self.advertised = false;
4528                self.next_probe_at = None;
4529                return;
4530            }
4531        };
4532
4533        let Some(registration) = registration else {
4534            self.registered_connection = None;
4535            self.advertised = false;
4536            self.next_probe_at = None;
4537            return;
4538        };
4539
4540        let advertised = registration
4541            .control_ops
4542            .iter()
4543            .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4544        if !advertised {
4545            self.registered_connection = Some(registration.connection_id);
4546            self.advertised = false;
4547            self.next_probe_at = None;
4548            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4549                if state.health.status != SupervisorHealthStatus::Unknown
4550                    || state.health.consecutive_failures != 0
4551                    || state.health.last_probe_ms.is_some()
4552                    || state.health.detail.is_some()
4553                    || state.health.metrics.is_some()
4554                {
4555                    state.health.status = SupervisorHealthStatus::Unknown;
4556                    state.health.consecutive_failures = 0;
4557                    state.health.last_probe_ms = None;
4558                    state.health.detail = None;
4559                    state.health.metrics = None;
4560                }
4561            });
4562            return;
4563        }
4564
4565        let reregistered = self.registered_connection != Some(registration.connection_id);
4566        self.registered_connection = Some(registration.connection_id);
4567        self.advertised = true;
4568        if reregistered || self.next_probe_at.is_none() {
4569            self.probe_index = 0;
4570            self.next_probe_at = Some(
4571                Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4572            );
4573            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4574                state.health.status = SupervisorHealthStatus::Unknown;
4575                state.health.consecutive_failures = 0;
4576                state.health.detail = None;
4577                state.health.metrics = None;
4578            });
4579        }
4580    }
4581
4582    fn wake_after(&self) -> Option<Duration> {
4583        if !self.advertised {
4584            return None;
4585        }
4586        self.next_probe_at
4587            .map(|next| next.saturating_duration_since(Instant::now()))
4588    }
4589
4590    fn due(&self) -> bool {
4591        self.advertised
4592            && self
4593                .next_probe_at
4594                .is_some_and(|next| Instant::now() >= next)
4595    }
4596
4597    fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4598        self.probe_index = self.probe_index.wrapping_add(1);
4599        self.next_probe_at = Some(
4600            Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4601        );
4602    }
4603}
4604
4605/// What a failed health probe actually OBSERVED, kept apart from how it reads.
4606///
4607/// This was a struct with a single `message: String`, and every one of the
4608/// fifteen construction sites collapsed into it. Each site knows exactly what it
4609/// saw -- the lane is gone, the module did not answer in time, the module
4610/// answered with the wrong thing -- and `handle_health_probe_failure` then
4611/// treated all of them identically: increment a counter, compare to a threshold,
4612/// restart the module. THE DISTINCTION EXISTED AT EVERY CALL SITE AND WAS
4613/// DESTROYED BEFORE THE DECISION THAT NEEDED IT.
4614///
4615/// The distinction that matters is not severity, it is EVIDENTIAL WEIGHT:
4616///
4617/// * `LaneDead` is PROOF. The module's control connection is gone; nothing will
4618///   answer on it again.
4619/// * `NoAnswer` is ABSENCE OF EVIDENCE. It is consistent with a wedged module
4620///   AND with a perfectly healthy one that lost a CPU race -- which is what
4621///   happens under machine load, and is how this supervisor killed a healthy
4622///   module three times in one day.
4623/// * `BadAnswer` proves the module is ALIVE. It replied; the reply was wrong.
4624///   Restarting on it is defensible, but it is not the silence case and should
4625///   never be counted as one.
4626/// * `Misconfigured` is a daemon-side fault. The module has not been asked
4627///   anything, so it cannot be evidence about the module at all.
4628///
4629/// The asymmetry is the whole point: under saturation the WEAKEST signal is the
4630/// one that fires most often, and while every variant collapsed into one string
4631/// it carried the same weight as the strongest.
4632///
4633/// LIVE BEHAVIOUR TODAY, stated here because this doc block describes the
4634/// DESIGN and a reader stopping at it gets the build backwards: the restart
4635/// decision does NOT yet consult this classification -- consecutive `NoAnswer`
4636/// probes still increment the failure streak and drive escalation at the
4637/// threshold (see `is_proof_of_death` below for why that is deliberate and
4638/// what gates the change). Absence of evidence restarts modules today.
4639#[derive(Debug)]
4640enum HealthProbeEvidence {
4641    /// The module's control lane is gone. Proof of death.
4642    LaneDead,
4643    /// No reply within the deadline. Proves nothing about the module's state.
4644    NoAnswer,
4645    /// The module replied, but not with a usable health report. Proves it is alive.
4646    BadAnswer,
4647    /// The daemon could not ask. Says nothing about the module.
4648    Misconfigured,
4649}
4650
4651#[derive(Debug)]
4652struct HealthProbeError {
4653    evidence: HealthProbeEvidence,
4654    message: String,
4655}
4656
4657impl HealthProbeError {
4658    fn lane_dead(message: impl Into<String>) -> Self {
4659        Self::with(HealthProbeEvidence::LaneDead, message)
4660    }
4661
4662    fn no_answer(message: impl Into<String>) -> Self {
4663        Self::with(HealthProbeEvidence::NoAnswer, message)
4664    }
4665
4666    fn bad_answer(message: impl Into<String>) -> Self {
4667        Self::with(HealthProbeEvidence::BadAnswer, message)
4668    }
4669
4670    fn misconfigured(message: impl Into<String>) -> Self {
4671        Self::with(HealthProbeEvidence::Misconfigured, message)
4672    }
4673
4674    fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4675        Self {
4676            evidence,
4677            message: message.into(),
4678        }
4679    }
4680
4681    /// Whether this observation is proof the module cannot serve.
4682    ///
4683    /// Only `LaneDead` qualifies. `NoAnswer` is deliberately excluded: it is the
4684    /// variant that fires under CPU starvation, and treating it as proof is the
4685    /// defect this enum exists to make impossible to reintroduce silently.
4686    ///
4687    /// NOT YET CONSULTED BY THE RESTART DECISION, deliberately. Requiring proof
4688    /// to restart also needs a bound for the case it excludes -- a genuinely
4689    /// wedged module, alive but never answering -- and that bound must come from
4690    /// the distribution of real late-answer latencies, which nothing measures
4691    /// yet. Landing the classification first makes the later change a one-line
4692    /// decision against evidence that already exists, rather than two unproven
4693    /// changes at once.
4694    #[allow(dead_code)]
4695    fn is_proof_of_death(&self) -> bool {
4696        matches!(self.evidence, HealthProbeEvidence::LaneDead)
4697    }
4698
4699    /// Short stable label for logs and the health snapshot.
4700    ///
4701    /// An operator reading `ck health` currently cannot tell "the module is gone"
4702    /// from "the module did not answer in five seconds", because both render as
4703    /// prose in the same field. These labels are what make the two
4704    /// distinguishable at a glance, and they are what a later restart-policy
4705    /// change will be argued from.
4706    fn label(&self) -> &'static str {
4707        match self.evidence {
4708            HealthProbeEvidence::LaneDead => "lane-dead",
4709            HealthProbeEvidence::NoAnswer => "no-answer",
4710            HealthProbeEvidence::BadAnswer => "bad-answer",
4711            HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4712        }
4713    }
4714}
4715
4716impl fmt::Display for HealthProbeError {
4717    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4718        f.write_str(&self.message)
4719    }
4720}
4721
4722async fn run_health_probe_cycle(
4723    spec: &ModuleSpec,
4724    runtime: &SupervisorRuntimeConfig,
4725    registry: &Registry,
4726    process_liveness: &SupervisorProcessLiveness,
4727    snapshot: &SharedSnapshot,
4728    child: &mut Option<SupervisedChild>,
4729) {
4730    let now_ms = unix_ms_now();
4731    let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4732        .then_some(runtime.health.http.as_deref())
4733        .flatten();
4734    let result = match http {
4735        Some(url) => probe_http_health(url, runtime.health.deadline).await,
4736        None => probe_module_health(&spec.module_id, runtime, None).await,
4737    };
4738    match result {
4739        Ok(report) => {
4740            handle_health_report(
4741                spec,
4742                runtime,
4743                registry,
4744                process_liveness,
4745                snapshot,
4746                child,
4747                report,
4748                now_ms,
4749            )
4750            .await;
4751        }
4752        Err(err) => {
4753            if http.is_some() {
4754                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4755                    state.health.status = SupervisorHealthStatus::Failing;
4756                });
4757            }
4758            handle_health_probe_failure(
4759                spec,
4760                runtime,
4761                registry,
4762                process_liveness,
4763                snapshot,
4764                child,
4765                err,
4766                now_ms,
4767            )
4768            .await;
4769        }
4770    }
4771}
4772
4773pub(crate) struct HttpProbeTarget<'a> {
4774    address: std::net::SocketAddr,
4775    localhost: bool,
4776    authority: &'a str,
4777    path: String,
4778}
4779
4780/// Resolve only literal loopback endpoints, without DNS, redirects, proxies,
4781/// or TLS. A URL cannot turn a local health check into an outbound connection.
4782pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4783    if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4784        return Err("must not contain whitespace, controls, or a fragment".into());
4785    }
4786    let rest = url
4787        .strip_prefix("http://")
4788        .ok_or("must use plain http://")?;
4789    let split = rest.find(['/', '?']).unwrap_or(rest.len());
4790    let (authority, suffix) = rest.split_at(split);
4791    let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4792        ("::1", rest)
4793    } else {
4794        let split = authority.find(':').unwrap_or(authority.len());
4795        authority.split_at(split)
4796    };
4797    let ip: std::net::IpAddr = match host {
4798        "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4799        "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4800        _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4801    };
4802    let port = if port.is_empty() {
4803        80
4804    } else {
4805        port.strip_prefix(':')
4806            .and_then(|p| p.parse::<u16>().ok())
4807            .filter(|p| *p > 0)
4808            .ok_or("must have a valid nonzero TCP port")?
4809    };
4810    let path = if suffix.is_empty() {
4811        "/".into()
4812    } else if suffix.starts_with('?') {
4813        format!("/{suffix}")
4814    } else {
4815        suffix.into()
4816    };
4817    Ok(HttpProbeTarget {
4818        address: std::net::SocketAddr::new(ip, port),
4819        localhost: host == "localhost",
4820        authority,
4821        path,
4822    })
4823}
4824
4825async fn probe_http_health(
4826    url: &str,
4827    deadline: Duration,
4828) -> Result<HealthReport, HealthProbeError> {
4829    use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4830    let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4831    // Keep partial diagnostics outside the timed future so cancellation does
4832    // not discard a status line or body bytes already received.
4833    let mut response_status = String::new();
4834    let mut body = Vec::new();
4835    let probe = async {
4836        // Resolve localhost ourselves so a hosts-file override cannot turn
4837        // this into an outbound request, while IPv6-only local servers work.
4838        let connection = match tokio::net::TcpStream::connect(target.address).await {
4839            Err(_) if target.localhost => {
4840                tokio::net::TcpStream::connect((
4841                    std::net::Ipv6Addr::LOCALHOST,
4842                    target.address.port(),
4843                ))
4844                .await
4845            }
4846            result => result,
4847        };
4848        let mut stream = connection.map_err(|error| {
4849            HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4850        })?;
4851        stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4852            .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4853        let mut reader = BufReader::new(stream);
4854        let mut budget = 16 * 1024;
4855        let status = http_line(&mut reader, &mut budget).await?;
4856        let mut words = status.split_ascii_whitespace();
4857        let version = words.next();
4858        let code = words
4859            .next()
4860            .filter(|word| word.len() == 3)
4861            .and_then(|word| word.parse::<u16>().ok());
4862        if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4863            || !code.is_some_and(|code| (100..600).contains(&code))
4864        {
4865            return Err(HealthProbeError::bad_answer(format!(
4866                "invalid HTTP status: {status}"
4867            )));
4868        }
4869        let code = code.expect("validated status code");
4870        response_status = status.clone();
4871        let mut length = None;
4872        let mut chunked = false;
4873        loop {
4874            let line = http_line(&mut reader, &mut budget).await?;
4875            if line.is_empty() {
4876                break;
4877            }
4878            if let Some((name, value)) = line.split_once(':') {
4879                if name.eq_ignore_ascii_case("content-length") {
4880                    length = Some(value.trim().parse::<u64>().map_err(|_| {
4881                        HealthProbeError::bad_answer("invalid HTTP Content-Length")
4882                    })?);
4883                } else if name.eq_ignore_ascii_case("transfer-encoding") {
4884                    chunked = value.trim().eq_ignore_ascii_case("chunked");
4885                }
4886            }
4887        }
4888        if chunked {
4889            while body.len() < 200 {
4890                let line = http_line(&mut reader, &mut budget).await?;
4891                let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4892                    .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4893                if size == 0 {
4894                    break;
4895                }
4896                let count = size.min((200 - body.len()) as u64) as usize;
4897                let start = body.len();
4898                (&mut reader)
4899                    .take(count as u64)
4900                    .read_to_end(&mut body)
4901                    .await
4902                    .map_err(|error| {
4903                        HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4904                    })?;
4905                if body.len() - start != count {
4906                    return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4907                }
4908                if size > count as u64 || body.len() == 200 {
4909                    break;
4910                }
4911                if !http_line(&mut reader, &mut budget).await?.is_empty() {
4912                    return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4913                }
4914            }
4915        } else {
4916            reader
4917                .take(length.unwrap_or(200).min(200))
4918                .read_to_end(&mut body)
4919                .await
4920                .map_err(|error| {
4921                    HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4922                })?;
4923        }
4924        if (200..300).contains(&code) {
4925            Ok(HealthReport::ok())
4926        } else {
4927            Err(HealthProbeError::bad_answer(
4928                "HTTP health endpoint returned non-2xx",
4929            ))
4930        }
4931    };
4932    let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4933        Err(HealthProbeError::no_answer(format!(
4934            "HTTP probe timed out after {deadline:?}"
4935        )))
4936    });
4937    if let Err(error) = &mut result {
4938        if !response_status.is_empty() {
4939            error.message = format!(
4940                "{}; {response_status}: {}",
4941                error.message,
4942                String::from_utf8_lossy(&body)
4943            );
4944        }
4945    }
4946    result
4947}
4948
4949async fn http_line(
4950    reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4951    remaining: &mut usize,
4952) -> Result<String, HealthProbeError> {
4953    use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4954    let mut line = Vec::new();
4955    (&mut *reader)
4956        .take(*remaining as u64)
4957        .read_until(b'\n', &mut line)
4958        .await
4959        .map_err(|error| {
4960            HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4961        })?;
4962    *remaining -= line.len();
4963    if !line.ends_with(b"\r\n") {
4964        return Err(HealthProbeError::bad_answer(
4965            "HTTP headers are incomplete or exceed 16 KiB",
4966        ));
4967    }
4968    line.truncate(line.len() - 2);
4969    String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4970}
4971
4972async fn probe_module_health(
4973    module_id: &str,
4974    runtime: &SupervisorRuntimeConfig,
4975    drain_deadline: Option<Instant>,
4976) -> Result<HealthReport, HealthProbeError> {
4977    let Some(forwarding) = runtime.forwarding.as_ref() else {
4978        return Err(HealthProbeError::misconfigured(
4979            "supervisor was not configured with a forwarding table",
4980        ));
4981    };
4982    let probe_started_at = Instant::now();
4983    let mut deadline = probe_started_at + runtime.health.deadline;
4984    if let Some(drain_deadline) = drain_deadline {
4985        deadline = deadline.min(drain_deadline);
4986    }
4987    let pending = if drain_deadline.is_some() {
4988        forwarding.begin_drain_health_probe_rpc_for(
4989            module_id,
4990            MODULE_CONTROL_OP_HEALTH_CHECK,
4991            probe_started_at,
4992            deadline,
4993        )
4994    } else {
4995        forwarding.begin_health_probe_rpc_for(
4996            module_id,
4997            MODULE_CONTROL_OP_HEALTH_CHECK,
4998            probe_started_at,
4999            deadline,
5000        )
5001    }
5002    .map_err(|err| {
5003        // The endpoint is not registered, so there is no live control lane to
5004        // ask. That is the module being absent, not slow.
5005        HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5006    })?;
5007    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5008}
5009
5010/// [`probe_module_health`] for one endpoint rather than the id's active one.
5011///
5012/// A swap probes two processes that no by-id lookup reaches: its candidate
5013/// before cutover, and its superseded incumbent (for busy gauges) while the
5014/// incumbent drains. `deadline_cap` bounds the probe the way a drain deadline
5015/// bounds the by-id drain probe.
5016async fn probe_endpoint_health(
5017    endpoint: crate::ModuleEndpointId,
5018    runtime: &SupervisorRuntimeConfig,
5019    deadline_cap: Option<Instant>,
5020) -> Result<HealthReport, HealthProbeError> {
5021    let Some(forwarding) = runtime.forwarding.as_ref() else {
5022        return Err(HealthProbeError::misconfigured(
5023            "supervisor was not configured with a forwarding table",
5024        ));
5025    };
5026    let probe_started_at = Instant::now();
5027    let mut deadline = probe_started_at + runtime.health.deadline;
5028    if let Some(cap) = deadline_cap {
5029        deadline = deadline.min(cap);
5030    }
5031    let pending = forwarding
5032        .begin_endpoint_health_probe_rpc_for(
5033            endpoint,
5034            MODULE_CONTROL_OP_HEALTH_CHECK,
5035            probe_started_at,
5036            deadline,
5037        )
5038        .map_err(|err| {
5039            HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5040        })?;
5041    await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5042}
5043
5044/// Send a begun health probe and classify its answer.
5045async fn await_health_probe(
5046    forwarding: &ForwardingTable,
5047    pending: PendingModuleControlRpc,
5048    deadline: Instant,
5049    probe_budget: Duration,
5050) -> Result<HealthReport, HealthProbeError> {
5051    let PendingModuleControlRpc {
5052        endpoint,
5053        module_sink,
5054        negotiated_ver,
5055        corr,
5056        receiver,
5057    } = pending;
5058    let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5059        HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5060    })?;
5061    let frame = Frame::build_with_version(
5062        negotiated_ver,
5063        FrameType::Request,
5064        control_flags(),
5065        0,
5066        0,
5067        corr,
5068        body,
5069    )
5070    .map_err(|err| {
5071        HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5072    })?;
5073
5074    // The enqueue itself must be bounded by the probe deadline: FrameSink.send
5075    // blocks waiting for capacity when the module's egress queue is full, and an
5076    // unbounded await here freezes the whole supervision actor (it stops polling
5077    // Child::wait and supervisor commands), making the module unrecoverable
5078    // in-band. On timeout the probe fails like any transport failure.
5079    match timeout_at(deadline, module_sink.send(frame)).await {
5080        Ok(Ok(())) => {}
5081        Ok(Err(err)) => {
5082            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5083            // A closed sink means the module's egress channel is gone -- the
5084            // receiving half is dropped when its connection tears down. Proof.
5085            return Err(HealthProbeError::lane_dead(format!(
5086                "failed to send health.check: {err}"
5087            )));
5088        }
5089        Err(_elapsed) => {
5090            let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5091            // A full egress queue means the module is not draining its socket, which
5092            // is consistent with a wedged module AND with one whose reader is merely
5093            // starved. Silence, not proof.
5094            return Err(HealthProbeError::no_answer(
5095                "health.check send timed out before enqueue (module egress full)",
5096            ));
5097        }
5098    }
5099
5100    match timeout_at(deadline, receiver).await {
5101        // Each arm records WHAT WAS OBSERVED. Four of them are the module
5102        // demonstrably answering -- rejected, non-health, malformed, wrong op --
5103        // and those prove it is alive even though the probe failed.
5104        Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5105            response.health_report().ok_or_else(|| {
5106                HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5107            })
5108        }
5109        Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5110            format!("health.check rejected: {}", body.message),
5111        )),
5112        Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5113            Err(HealthProbeError::lane_dead(message))
5114        }
5115        Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5116            Err(HealthProbeError::bad_answer(message))
5117        }
5118        Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5119            Err(HealthProbeError::bad_answer(format!(
5120                "expected module-control op '{expected}', got '{actual}'"
5121            )))
5122        }
5123        // A reply that crosses the deadline before this waiter observes it is
5124        // still proof of life. The forwarding path records its end-to-end latency
5125        // before delivering this classification.
5126        Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5127            "module answered health.check after its daemon deadline",
5128        )),
5129        Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5130            "health.check waiter was canceled before the module responded",
5131        )),
5132        Err(_) => {
5133            let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5134            Err(HealthProbeError::no_answer(format!(
5135                "module did not answer health.check within {probe_budget:?}"
5136            )))
5137        }
5138    }
5139}
5140
5141#[allow(clippy::too_many_arguments)]
5142async fn handle_health_report(
5143    spec: &ModuleSpec,
5144    runtime: &SupervisorRuntimeConfig,
5145    registry: &Registry,
5146    process_liveness: &SupervisorProcessLiveness,
5147    snapshot: &SharedSnapshot,
5148    child: &mut Option<SupervisedChild>,
5149    report: HealthReport,
5150    now_ms: u64,
5151) {
5152    let status = supervisor_health_status(report.status);
5153    let detail = report.detail.clone();
5154    let metrics = truncate_health_metrics(report.metrics);
5155    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5156        state.health.status = status;
5157        state.health.last_probe_ms = Some(now_ms);
5158        state.health.detail = detail.clone();
5159        state.health.metrics = metrics.clone();
5160        state.health.consecutive_failures = 0;
5161    });
5162
5163    let action = match report.status {
5164        HealthStatus::Ok => return,
5165        HealthStatus::Degraded => runtime.health.on_degraded,
5166        HealthStatus::Failing => runtime.health.on_failing,
5167    };
5168    apply_l3_health_action(
5169        spec,
5170        runtime,
5171        registry,
5172        process_liveness,
5173        snapshot,
5174        child,
5175        status,
5176        detail.as_deref(),
5177        action,
5178        now_ms,
5179    )
5180    .await;
5181}
5182
5183#[allow(clippy::too_many_arguments)]
5184async fn handle_health_probe_failure(
5185    spec: &ModuleSpec,
5186    runtime: &SupervisorRuntimeConfig,
5187    registry: &Registry,
5188    process_liveness: &SupervisorProcessLiveness,
5189    snapshot: &SharedSnapshot,
5190    child: &mut Option<SupervisedChild>,
5191    err: HealthProbeError,
5192    now_ms: u64,
5193) {
5194    let threshold = runtime.health.failure_threshold.max(1);
5195    let mut failures = 0;
5196    // Carry the evidence class into the operator-visible detail. Without it,
5197    // "module did not answer within 5s" and "the control lane is gone" are two
5198    // prose strings in the same field, and the reader has to know the codebase to
5199    // tell which one is proof of anything.
5200    let detail = format!("[{}] {err}", err.label());
5201    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5202        // A failed wire probe invalidates the last report, even before the
5203        // restart threshold. HTTP probes already mark failures as Failing.
5204        if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5205            state.health.status = SupervisorHealthStatus::Unknown;
5206        }
5207        state.health.last_probe_ms = Some(now_ms);
5208        state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5209        state.health.detail = Some(detail.clone());
5210        state.health.metrics = None;
5211        failures = state.health.consecutive_failures;
5212    });
5213
5214    if failures < threshold {
5215        warn!(
5216            module_id = %spec.module_id,
5217            consecutive_failures = failures,
5218            threshold,
5219            evidence = err.label(),
5220            detail = %detail,
5221            "health.check probe failed"
5222        );
5223        return;
5224    }
5225
5226    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5227        state.state = ModuleState::Unresponsive;
5228        state.health.status = SupervisorHealthStatus::Unresponsive;
5229    });
5230    // The evidence class is logged at the kill site because this is the line an
5231    // operator reads after an unexplained restart. A streak of `no-answer` under
5232    // machine load is the known false-positive shape; a `lane-dead` is not.
5233    if runtime.health.critical {
5234        error!(
5235            module_id = %spec.module_id,
5236            status = "unresponsive",
5237            evidence = err.label(),
5238            detail = %detail,
5239            "critical module health alert"
5240        );
5241    } else {
5242        warn!(
5243            module_id = %spec.module_id,
5244            status = "unresponsive",
5245            evidence = err.label(),
5246            detail = %detail,
5247            "module health threshold breached"
5248        );
5249    }
5250    if let Err(err) = health_restart_child(
5251        spec,
5252        runtime,
5253        registry,
5254        process_liveness,
5255        snapshot,
5256        child,
5257        SupervisorHealthStatus::Unresponsive,
5258        Some(&detail),
5259        now_ms,
5260    )
5261    .await
5262    {
5263        error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5264    }
5265}
5266
5267#[allow(clippy::too_many_arguments)]
5268async fn apply_l3_health_action(
5269    spec: &ModuleSpec,
5270    runtime: &SupervisorRuntimeConfig,
5271    registry: &Registry,
5272    process_liveness: &SupervisorProcessLiveness,
5273    snapshot: &SharedSnapshot,
5274    child: &mut Option<SupervisedChild>,
5275    status: SupervisorHealthStatus,
5276    detail: Option<&str>,
5277    action: HealthAction,
5278    now_ms: u64,
5279) {
5280    record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5281    match action {
5282        HealthAction::Report => {
5283            info!(
5284                module_id = %spec.module_id,
5285                status = ?status,
5286                detail,
5287                "module reported non-ok health"
5288            );
5289        }
5290        HealthAction::Alert => {
5291            error!(
5292                module_id = %spec.module_id,
5293                status = ?status,
5294                detail,
5295                "module health alert"
5296            );
5297        }
5298        HealthAction::Restart => {
5299            if let Err(err) = health_restart_child(
5300                spec,
5301                runtime,
5302                registry,
5303                process_liveness,
5304                snapshot,
5305                child,
5306                status,
5307                detail,
5308                now_ms,
5309            )
5310            .await
5311            {
5312                error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5313            }
5314        }
5315    }
5316}
5317
5318#[allow(clippy::too_many_arguments)]
5319async fn health_restart_child(
5320    spec: &ModuleSpec,
5321    runtime: &SupervisorRuntimeConfig,
5322    registry: &Registry,
5323    process_liveness: &SupervisorProcessLiveness,
5324    snapshot: &SharedSnapshot,
5325    child: &mut Option<SupervisedChild>,
5326    status: SupervisorHealthStatus,
5327    detail: Option<&str>,
5328    now_ms: u64,
5329) -> Result<(), SuperviseError> {
5330    let (enabled, schedule) = {
5331        let mut state = lock_snapshot(snapshot)?;
5332        let enabled = state.enabled;
5333        let schedule = if enabled {
5334            state.next_crash_restart(&runtime.restart_policy, Instant::now())
5335        } else {
5336            None
5337        };
5338        (enabled, schedule)
5339    };
5340
5341    if !enabled {
5342        return Err(SuperviseError::Disabled {
5343            module_id: spec.module_id.clone(),
5344        });
5345    }
5346
5347    if schedule.is_none() {
5348        record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5349        error!(
5350            module_id = %spec.module_id,
5351            status = ?status,
5352            detail,
5353            max_restarts = runtime.restart_policy.max_restarts,
5354            window_secs = runtime.restart_policy.window.as_secs(),
5355            reason = %runtime.restart_policy.budget_exhausted_detail(),
5356            "health restart budget exhausted; marking module failed"
5357        );
5358        let stop_notice = begin_forwarding_drain_if_configured(
5359            spec,
5360            runtime,
5361            registry,
5362            snapshot,
5363            Some(true),
5364            RouteCloseReason::Disable,
5365        )
5366        .await?;
5367        update_snapshot(snapshot, Some(&spec.module_id), |state| {
5368            state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5369        })?;
5370        drain_optional_child(
5371            &spec.module_id,
5372            spec.protocol,
5373            stop_notice,
5374            registry,
5375            runtime.forwarding.as_deref(),
5376            snapshot,
5377            &runtime.terminal_ring,
5378            &runtime.spawn_events,
5379            child,
5380            runtime.drain_timeout,
5381            ModuleState::Failed,
5382            Some(true),
5383        )
5384        .await?;
5385        process_liveness.untrack_if_current(&spec.module_id, snapshot);
5386        return Ok(());
5387    }
5388
5389    let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5390    let mut restart_count = 0;
5391    update_snapshot(snapshot, Some(&spec.module_id), |state| {
5392        restart_count = state.crash_restarts.len();
5393        state.state = ModuleState::Unresponsive;
5394        state.health.status = status;
5395        state.health.last_action = Some(HealthAction::Restart.to_string());
5396        state.health.last_action_ms = Some(now_ms);
5397    })?;
5398    warn!(
5399        module_id = %spec.module_id,
5400        status = ?status,
5401        detail,
5402        restart_count,
5403        restart_in_window = schedule.restart_in_window,
5404        delay_ms = schedule.delay.as_millis() as u64,
5405        "health-triggered module restart"
5406    );
5407
5408    let stop_notice = begin_forwarding_drain_if_configured(
5409        spec,
5410        runtime,
5411        registry,
5412        snapshot,
5413        Some(true),
5414        RouteCloseReason::Restart,
5415    )
5416    .await?;
5417    drain_optional_child(
5418        &spec.module_id,
5419        spec.protocol,
5420        stop_notice,
5421        registry,
5422        runtime.forwarding.as_deref(),
5423        snapshot,
5424        &runtime.terminal_ring,
5425        &runtime.spawn_events,
5426        child,
5427        runtime.drain_timeout,
5428        ModuleState::Restarting,
5429        Some(true),
5430    )
5431    .await?;
5432    schedule_respawn(
5433        runtime,
5434        snapshot,
5435        &spec.module_id,
5436        schedule.delay,
5437        RespawnKind::Spawn,
5438    )
5439}
5440
5441fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5442    if let Some(reply) = runtime
5443        .deferred_reload_reply
5444        .lock()
5445        .unwrap_or_else(|p| p.into_inner())
5446        .take()
5447    {
5448        let _ = reply.send(Err(SuperviseError::ReloadFailed {
5449            module_id: module_id.to_string(),
5450            reason: reason.to_string(),
5451        }));
5452    }
5453}
5454
5455fn schedule_respawn(
5456    runtime: &SupervisorRuntimeConfig,
5457    snapshot: &SharedSnapshot,
5458    module_id: &str,
5459    delay: Duration,
5460    kind: RespawnKind,
5461) -> Result<(), SuperviseError> {
5462    cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5463    update_snapshot(snapshot, Some(module_id), |state| {
5464        state.respawn_pending = true
5465    })?;
5466    *runtime
5467        .scheduled_respawn
5468        .lock()
5469        .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5470        deadline: Instant::now() + delay,
5471        kind,
5472    });
5473    Ok(())
5474}
5475
5476fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5477    let _ = update_snapshot(snapshot, Some(module_id), |state| {
5478        state.health.last_action = Some(action);
5479        state.health.last_action_ms = Some(now_ms);
5480    });
5481}
5482
5483fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5484    match status {
5485        HealthStatus::Ok => SupervisorHealthStatus::Ok,
5486        HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5487        HealthStatus::Failing => SupervisorHealthStatus::Failing,
5488    }
5489}
5490
5491/// Caps the metrics blob stored in the cached supervisor snapshot, which is
5492/// returned to every `supervisor.list` and `supervisor.health` caller.
5493///
5494/// This cap is deliberately NOT applied on the one-shot `supervisor.health_probe`
5495/// path: that request exists to return a module's complete metrics object, and
5496/// `ck health <module-id>` documents it as the way to see what the cached view
5497/// truncates. The asymmetry is the feature.
5498///
5499/// So a new caller must decide which side it is on rather than assume the cap is
5500/// universal. Reaching for it on a fresh-probe path would silently reintroduce
5501/// the truncation that path exists to avoid.
5502fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5503    let metrics = metrics?;
5504    match serde_json::to_vec(&metrics) {
5505        Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5506            "truncated": true,
5507            "original_bytes": encoded.len(),
5508        })),
5509        Ok(_) | Err(_) => Some(metrics),
5510    }
5511}
5512
5513/// Spread health probes so a fleet-wide restart does not converge them.
5514///
5515/// The delay is derived from the module id and probe index rather than a random
5516/// source, so it is deterministic per module: a module keeps its own offset
5517/// across daemon restarts instead of re-rolling into a collision.
5518fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5519    if cadence.is_zero() {
5520        return Duration::ZERO;
5521    }
5522    let cadence_ms = cadence.as_millis() as u64;
5523    // This early return is REDUNDANT, deliberately, and a mutation run will show
5524    // it surviving removal. Recording why here so the next person to notice does
5525    // not have to re-derive it:
5526    //
5527    // - It is unreachable in practice. `positive_millis` in daemon_config rejects
5528    //   a zero cadence and builds the Duration from whole milliseconds, so a
5529    //   sub-millisecond cadence cannot come from config.
5530    // - Even if reached it changes no answer. The `.max(1)` below makes the span
5531    //   1, and `hash % 1` is 0, so the fall-through returns `cadence` unchanged
5532    //   -- exactly what this returns.
5533    //
5534    // Kept as a guard against a future widening of the config parser (accepting
5535    // microseconds, say), which would make the sub-millisecond case reachable.
5536    // The `.max(1)` is the load-bearing half TODAY: remove it and the modulo
5537    // divides by zero. Remove this and nothing changes.
5538    if cadence_ms == 0 {
5539        return cadence;
5540    }
5541    // Note that this never returns less than one cadence, including for the FIRST
5542    // probe. So a freshly registered module reports health `unknown` for a full
5543    // cadence plus jitter -- 30-33s at the default -- no matter how quickly it is
5544    // ready to answer.
5545    //
5546    // That is a property of the supervisor's schedule, not of any module: an
5547    // operator watching a restart sees `unknown` and cannot tell it from a module
5548    // that is slow to warm. Measured on two unrelated modules, both flipping to
5549    // `ok` between 22s and 32s after restart.
5550    //
5551    // Left as-is because spreading the first probe is what keeps a fleet-wide
5552    // restart from firing fourteen simultaneous probes into a cold machine. The
5553    // alternative -- probe at t+0 and jitter only from the second onward -- trades
5554    // that thundering herd for a faster first reading.
5555    let jitter_span = (cadence_ms / 10).max(1);
5556    let hash = module_id.as_bytes().iter().fold(
5557        probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5558        |acc, byte| {
5559            acc.wrapping_mul(1099511628211)
5560                .wrapping_add(u64::from(*byte))
5561        },
5562    );
5563    cadence + Duration::from_millis(hash % jitter_span)
5564}
5565
5566#[cfg(test)]
5567mod tests {
5568    use super::*;
5569
5570    #[test]
5571    fn readding_a_module_clears_its_rescan_removal_tombstone() {
5572        let handle = SupervisorHandle::new();
5573        let module_id = "readded-tombstone";
5574        handle.record_rescan_removal(module_id);
5575        assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5576
5577        handle.apply_identity_configuration(&ModuleSpec {
5578            module_id: module_id.to_string(),
5579            program: PathBuf::from("/test/module"),
5580            args: Vec::new(),
5581            env: Vec::new(),
5582            reserved: false,
5583            reserved_prefixes: Vec::new(),
5584            protocol: ModuleProtocol::Subc,
5585            overlap: Default::default(),
5586        });
5587
5588        assert!(
5589            handle.removal_tombstone_age_ms(module_id).is_none(),
5590            "a re-added module must not retain a stale removal tombstone"
5591        );
5592    }
5593
5594    /// What one module's owner looked like from the control plane at the
5595    /// instant after its first process was spawned.
5596    #[derive(Debug, PartialEq, Eq)]
5597    struct OwnerInSpawnWindow {
5598        module_id: String,
5599        configured: bool,
5600        on_roster: bool,
5601        admission_refusal: Option<&'static str>,
5602    }
5603
5604    /// A supervised module's process can connect, register, sync its scopes
5605    /// and describe them as soon as it is spawned. A plain `spawn` does not put
5606    /// the module on the roster until afterward; configured supervision does
5607    /// so before spawning. In both cases the owner must
5608    /// already read as configured, so a scoped `route.open` against it is
5609    /// refused as retryable `scope_not_synced` and not as terminal
5610    /// `scope_not_live` ("will never sync").
5611    ///
5612    /// The hook runs just after each initial spawn attempt, before returning to
5613    /// the caller, so no race with a real child's registration is needed.
5614    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5615    async fn a_new_module_is_configured_before_its_first_process_can_register() {
5616        use crate::scopes::ScopeTable;
5617        use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5618
5619        let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5620        let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5621            module_id: module_id.to_string(),
5622            program,
5623            args: Vec::new(),
5624            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5625                .into_iter()
5626                .map(|key| (key.to_string(), dir.path().display().to_string()))
5627                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5628                .collect(),
5629            reserved: false,
5630            reserved_prefixes: Vec::new(),
5631            protocol: ModuleProtocol::Subc,
5632            overlap: Default::default(),
5633        };
5634        let live = super::terminal_history_tests::fake_aft_stub_path();
5635        let missing = dir.path().join("definitely-missing-module");
5636
5637        let handle = SupervisorHandle::new();
5638        let mut supervisor =
5639            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5640                .with_handle(handle.clone());
5641        let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5642        let hook_handle = handle.clone();
5643        let hook_observed = Arc::clone(&observed);
5644        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5645            // Exactly what the control plane computes for a scoped route.open
5646            // naming this module as the owner of a scope it has not synced.
5647            let configured = hook_handle.is_configured(module_id);
5648            let selector = ScopeSelector::new(
5649                Principal::Reserved {
5650                    module_id: module_id.to_string(),
5651                },
5652                "s".to_string(),
5653                Some(1),
5654            );
5655            let carrier = Principal::Reserved {
5656                module_id: "carrier".to_string(),
5657            };
5658            let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5659                .admit(&carrier, module_id, &selector, configured)
5660            {
5661                Ok(_) => None,
5662                Err(refusal) => Some(refusal.code),
5663            };
5664            hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5665                module_id: module_id.to_string(),
5666                configured,
5667                on_roster: hook_handle.get(module_id).is_some(),
5668                admission_refusal,
5669            });
5670        })));
5671
5672        let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5673        let configured = supervisor
5674            .supervise_configured(stub("configured", live.clone()), true)
5675            .unwrap();
5676        let with_health = supervisor
5677            .supervise_configured_with_health(
5678                stub("with-health", live.clone()),
5679                true,
5680                HealthConfig::default(),
5681                None,
5682                RestartPolicy::default(),
5683            )
5684            .unwrap();
5685        // The failed-spawn path still puts the module on the roster (as
5686        // failed), so it is configured throughout.
5687        let failed = supervisor
5688            .supervise_configured_with_health(
5689                stub("failed-spawn", missing.clone()),
5690                true,
5691                HealthConfig::default(),
5692                None,
5693                RestartPolicy::default(),
5694            )
5695            .unwrap();
5696        // A failed plain `spawn` puts nothing on the roster, so its mark is
5697        // taken back once the spawn has failed.
5698        assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5699
5700        let expected = [
5701            "plain",
5702            "configured",
5703            "with-health",
5704            "failed-spawn",
5705            "spawn-error",
5706        ]
5707        .into_iter()
5708        .map(|module_id| OwnerInSpawnWindow {
5709            module_id: module_id.to_string(),
5710            configured: true,
5711            on_roster: !matches!(module_id, "plain" | "spawn-error"),
5712            admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5713        })
5714        .collect::<Vec<_>>();
5715        assert_eq!(*observed.lock().unwrap(), expected);
5716
5717        for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5718            assert!(
5719                handle.get(module_id).is_some(),
5720                "{module_id} is on the roster"
5721            );
5722            assert!(
5723                handle.is_configured(module_id),
5724                "{module_id} stays configured"
5725            );
5726        }
5727        assert!(handle.get("spawn-error").is_none());
5728        assert!(
5729            !handle.is_configured("spawn-error"),
5730            "a plain spawn that failed must not leave its module marked configured"
5731        );
5732
5733        // Leaving the roster clears the mark with it.
5734        handle.retire("failed-spawn");
5735        assert!(!handle.is_configured("failed-spawn"));
5736
5737        for module in [plain, configured, with_health] {
5738            module.stop().await.unwrap();
5739        }
5740        drop(failed);
5741    }
5742
5743    /// A configured module reads as starting until the supervisor has admitted
5744    /// its first process (exec confirmed, identity durably recorded), even
5745    /// though that process is already running by then and can register over
5746    /// HELLO, which is what lists it in catalog.list. So a client that needs
5747    /// the supervisor state must wait for that state; seeing the module in
5748    /// the catalog does not mean supervisor.list already says running.
5749    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5750    async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5751        let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5752        let spec = ModuleSpec {
5753            module_id: "spawned-not-admitted".to_string(),
5754            program: super::terminal_history_tests::fake_aft_stub_path(),
5755            args: Vec::new(),
5756            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5757                .into_iter()
5758                .map(|key| (key.to_string(), dir.path().display().to_string()))
5759                .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5760                .collect(),
5761            reserved: false,
5762            reserved_prefixes: Vec::new(),
5763            protocol: ModuleProtocol::Subc,
5764            overlap: Default::default(),
5765        };
5766        let handle = SupervisorHandle::new();
5767        let mut supervisor =
5768            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5769                .with_handle(handle.clone());
5770        let observed = Arc::new(Mutex::new(None));
5771        let hook_handle = handle.clone();
5772        let hook_observed = Arc::clone(&observed);
5773        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5774            let state = hook_handle
5775                .get(module_id)
5776                .expect("a configured module is on the roster before it is spawned")
5777                .state()
5778                .unwrap();
5779            *hook_observed.lock().unwrap() = Some(state);
5780        })));
5781
5782        let module = supervisor.supervise_configured(spec, true).unwrap();
5783        assert_eq!(
5784            *observed.lock().unwrap(),
5785            Some(ModuleState::Starting),
5786            "between the spawn and its admission the supervisor still reports starting"
5787        );
5788        assert_eq!(module.state().unwrap(), ModuleState::Running);
5789        module.stop().await.unwrap();
5790    }
5791
5792    fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5793        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5794        update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5795            snapshot.process_alive = true;
5796            snapshot.pid = Some(41);
5797            snapshot.spawned_at_ms = Some(42);
5798            snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5799            snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5800            snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5801                device: 43,
5802                inode: 44,
5803                #[cfg(windows)]
5804                windows_image: None,
5805            });
5806        })
5807        .unwrap();
5808        snapshot
5809    }
5810
5811    fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5812        let snapshot = lock_snapshot(snapshot).unwrap();
5813        assert!(!snapshot.process_alive);
5814        assert_eq!(snapshot.pid, None);
5815        assert_eq!(snapshot.spawned_at_ms, None);
5816        assert_eq!(snapshot.spawned_from, None);
5817        assert_eq!(snapshot.launch_spec_digest, None);
5818        assert_eq!(snapshot.spawned_file_identity, None);
5819    }
5820
5821    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5822    async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5823        let supervisor =
5824            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5825        let mut runtime = supervisor.runtime_config();
5826        runtime.test_seed_stale_facts_before_enable_spawn = true;
5827        let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5828        let mut child = None;
5829        let spec = ModuleSpec {
5830            module_id: "failed-enable-clears-facts".to_string(),
5831            program: PathBuf::from("/definitely/missing/failed-enable-module"),
5832            args: Vec::new(),
5833            env: Vec::new(),
5834            reserved: false,
5835            reserved_prefixes: Vec::new(),
5836            protocol: ModuleProtocol::Subc,
5837            overlap: Default::default(),
5838        };
5839
5840        let result = set_child_enabled(
5841            &spec,
5842            &runtime,
5843            &supervisor.registry,
5844            &supervisor.process_liveness,
5845            &snapshot,
5846            &mut child,
5847            true,
5848        )
5849        .await;
5850
5851        assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5852        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5853        assert_snapshot_process_facts_cleared(&snapshot);
5854    }
5855
5856    #[tokio::test]
5857    async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5858        let supervisor =
5859            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5860        let runtime = supervisor.runtime_config();
5861        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5862            ModuleState::Restarting,
5863            true,
5864        )));
5865        let spec = ModuleSpec {
5866            module_id: "start-stranded-restarting".to_string(),
5867            program: super::terminal_history_tests::fake_aft_stub_path(),
5868            args: Vec::new(),
5869            env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5870            reserved: false,
5871            reserved_prefixes: Vec::new(),
5872            protocol: ModuleProtocol::None,
5873            overlap: Default::default(),
5874        };
5875        let mut child = None;
5876        lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5877        assert!(!super::set_child_enabled(
5878            &spec,
5879            &runtime,
5880            &Registry::default(),
5881            &supervisor.process_liveness,
5882            &snapshot,
5883            &mut child,
5884            true
5885        )
5886        .await
5887        .unwrap());
5888        assert!(child.is_none());
5889        lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5890        assert!(super::set_child_enabled(
5891            &spec,
5892            &runtime,
5893            &Registry::default(),
5894            &supervisor.process_liveness,
5895            &snapshot,
5896            &mut child,
5897            true
5898        )
5899        .await
5900        .unwrap());
5901        assert_eq!(
5902            lock_snapshot(&snapshot).unwrap().state,
5903            ModuleState::Running
5904        );
5905        let mut child = child.unwrap();
5906        child.start_kill().unwrap();
5907        child.wait().await.unwrap();
5908    }
5909
5910    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5911    async fn failed_reload_spawn_clears_current_process_facts() {
5912        let supervisor =
5913            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5914        let mut runtime = supervisor.runtime_config();
5915        runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5916        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5917        let mut child = None;
5918        let spec = ModuleSpec {
5919            module_id: "failed-reload-clears-facts".to_string(),
5920            program: PathBuf::from("/unused/failed-reload-module"),
5921            args: Vec::new(),
5922            env: Vec::new(),
5923            reserved: false,
5924            reserved_prefixes: Vec::new(),
5925            protocol: ModuleProtocol::Subc,
5926            overlap: Default::default(),
5927        };
5928
5929        let result = handle_reload_spawn_failure(
5930            &spec,
5931            &runtime,
5932            &supervisor.process_liveness,
5933            &snapshot,
5934            &mut child,
5935            "forced reload spawn failure".to_string(),
5936        )
5937        .await;
5938
5939        assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5940        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5941        assert_snapshot_process_facts_cleared(&snapshot);
5942    }
5943
5944    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5945    async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5946        let supervisor =
5947            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5948        let snapshot = stale_process_snapshot(ModuleState::Running, true);
5949        let module = supervisor.supervised_module(
5950            ModuleSpec {
5951                module_id: "drop-clears-facts".to_string(),
5952                program: PathBuf::from("/unused/drop-module"),
5953                args: Vec::new(),
5954                env: Vec::new(),
5955                reserved: false,
5956                reserved_prefixes: Vec::new(),
5957                protocol: ModuleProtocol::Subc,
5958                overlap: Default::default(),
5959            },
5960            supervisor.runtime_config(),
5961            Arc::clone(&snapshot),
5962            None,
5963        );
5964        assert!(!module
5965            .inner
5966            .monitor
5967            .lock()
5968            .unwrap()
5969            .as_ref()
5970            .unwrap()
5971            .is_finished());
5972
5973        drop(module);
5974
5975        assert_eq!(
5976            lock_snapshot(&snapshot).unwrap().state,
5977            ModuleState::Stopped
5978        );
5979        assert_snapshot_process_facts_cleared(&snapshot);
5980    }
5981
5982    #[cfg(unix)]
5983    #[tokio::test]
5984    async fn rescan_preserves_running_protocol_until_respawn() {
5985        let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5986        let initial = ModuleSpec {
5987            module_id: "rescan-protocol".into(),
5988            program: PathBuf::from("/bin/sleep"),
5989            args: vec!["60".into()],
5990            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5991                .into_iter()
5992                .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5993                .collect(),
5994            reserved: false,
5995            reserved_prefixes: vec![],
5996            protocol: ModuleProtocol::None,
5997            overlap: Default::default(),
5998        };
5999        let supervisor =
6000            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6001        let module = supervisor.spawn(initial.clone()).unwrap();
6002        assert!(module.status().unwrap().live);
6003        let mut next = initial;
6004        next.protocol = ModuleProtocol::Subc;
6005        module
6006            .update_configuration(next.clone(), HealthConfig::default(), None)
6007            .await
6008            .unwrap();
6009        assert!(
6010            module.status().unwrap().live,
6011            "rescan must not require HELLO from the old non-wire process"
6012        );
6013        let runtime = supervisor.runtime_config();
6014        let action = on_child_exit(
6015            &next,
6016            RestartPolicy::default(),
6017            &supervisor.registry,
6018            &module.inner.snapshot,
6019            &runtime.terminal_ring,
6020            &runtime.spawn_events,
6021            &runtime.child_roster,
6022            ExitReport {
6023                kind: ExitKind::Clean,
6024                code: Some(0),
6025                signal: None,
6026                at_ms: unix_ms_now(),
6027            },
6028        )
6029        .await;
6030        assert!(
6031            matches!(action, NextAction::Restart { .. }),
6032            "the old non-wire process's clean exit must restart"
6033        );
6034        module.drain().await.unwrap();
6035    }
6036
6037    #[cfg(unix)]
6038    #[tokio::test]
6039    async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6040        use std::os::unix::fs::PermissionsExt;
6041        let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6042        let script = dir.join("module.sh");
6043        std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6044        std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6045        let record_path = dir.join("live-children.json");
6046        let supervisor =
6047            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6048                .with_live_children_record(&record_path);
6049        for (program, args) in [
6050            (PathBuf::from("sleep"), vec!["60".into()]),
6051            (script, vec![]),
6052        ] {
6053            let spec = ModuleSpec {
6054                module_id: "image-identity".into(),
6055                program: program.clone(),
6056                args,
6057                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6058                    .into_iter()
6059                    .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6060                    .collect(),
6061                reserved: false,
6062                reserved_prefixes: vec![],
6063                protocol: ModuleProtocol::None,
6064                overlap: Default::default(),
6065            };
6066            let module = supervisor.spawn(spec).unwrap();
6067            #[cfg(target_os = "macos")]
6068            {
6069                // SETEXEC confirmation is asynchronous; the orphan record must
6070                // identify the final image, never the intermediate trampoline.
6071                let deadline = Instant::now() + Duration::from_secs(5);
6072                while crate::live_children::read_record(&record_path)
6073                    .unwrap()
6074                    .iter()
6075                    .all(|entry| entry.executable.is_none())
6076                {
6077                    assert!(Instant::now() < deadline, "module image was not confirmed");
6078                    tokio::time::sleep(Duration::from_millis(5)).await;
6079                }
6080            }
6081            let entry = crate::live_children::read_record(&record_path)
6082                .unwrap()
6083                .pop()
6084                .unwrap();
6085            let observed = subc_os::Process::open(entry.pid)
6086                .unwrap()
6087                .unwrap()
6088                .observe()
6089                .unwrap();
6090            let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6091            module.drain().await.unwrap();
6092            assert_eq!(
6093                verdict,
6094                crate::live_children::IdentityVerdict::Matches,
6095                "program {program:?}: recorded {entry:?}, observed {observed:?}"
6096            );
6097        }
6098    }
6099
6100    #[cfg(unix)]
6101    fn http_fixture(
6102        dir: &std::path::Path,
6103        url: &str,
6104        threshold: u32,
6105    ) -> crate::daemon_config::ConfiguredModule {
6106        let path = dir.join("subc.jsonc");
6107        std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6108            "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6109            "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6110            "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6111            "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6112        }}}).to_string()).unwrap();
6113        crate::daemon_config::load(&path)
6114            .unwrap()
6115            .unwrap()
6116            .modules
6117            .pop()
6118            .unwrap()
6119    }
6120
6121    #[cfg(unix)]
6122    async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6123        timeout(Duration::from_secs(5), async {
6124            loop {
6125                if module.status().unwrap().health.status == status {
6126                    break;
6127                }
6128                sleep(Duration::from_millis(5)).await;
6129            }
6130        })
6131        .await
6132        .unwrap_or_else(|_| {
6133            panic!(
6134                "expected {status:?}, got {:?}",
6135                module.status().unwrap().health
6136            )
6137        });
6138    }
6139
6140    #[cfg(unix)]
6141    #[tokio::test]
6142    async fn http_health_status_flips_ok_failing_ok() {
6143        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6144        let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6145        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6146        let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6147        let serving_status = status.clone();
6148        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6149        let server = tokio::spawn(async move {
6150            loop {
6151                let (mut stream, _) = listener.accept().await.unwrap();
6152                let mut request = [0u8; 2048];
6153                let count = stream.read(&mut request).await.unwrap();
6154                assert!(count > 0, "a probe must send an HTTP request");
6155                let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6156                let body = if code == 200 {
6157                    "ready"
6158                } else {
6159                    "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6160                };
6161                let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6162                let _ = stream.write_all(response.as_bytes()).await;
6163            }
6164        });
6165        let configured = http_fixture(&dir, &url, 1000);
6166        let module =
6167            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6168                .supervise_configured_with_health(
6169                    configured.module_spec(),
6170                    true,
6171                    configured.health,
6172                    configured.drain_timeout_ms,
6173                    configured.restart,
6174                )
6175                .unwrap();
6176        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6177        status.store(503, std::sync::atomic::Ordering::SeqCst);
6178        wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6179        assert!(module
6180            .status()
6181            .unwrap()
6182            .health
6183            .detail
6184            .unwrap()
6185            .contains("scratch failure"));
6186        status.store(200, std::sync::atomic::Ordering::SeqCst);
6187        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6188        assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6189        let before = module.status().unwrap();
6190        let (spec, mut health) = module.configuration().unwrap();
6191        health.http = None;
6192        module
6193            .update_configuration(spec.clone(), health.clone(), Some(10))
6194            .await
6195            .unwrap();
6196        wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6197        health.http = Some(url);
6198        module
6199            .update_configuration(spec, health, Some(10))
6200            .await
6201            .unwrap();
6202        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6203        assert_eq!(
6204            module.status().unwrap().pid,
6205            before.pid,
6206            "changing a probe must apply live, not restart its process"
6207        );
6208        let (spec, mut health) = module.configuration().unwrap();
6209        health.failure_threshold = 2;
6210        module
6211            .update_configuration(spec, health, Some(10))
6212            .await
6213            .unwrap();
6214        status.store(503, std::sync::atomic::Ordering::SeqCst);
6215        timeout(Duration::from_secs(5), async {
6216            while module.status().unwrap().spawn_generation == before.spawn_generation {
6217                sleep(Duration::from_millis(5)).await;
6218            }
6219        })
6220        .await
6221        .expect("sustained HTTP 503 responses must trigger the health restart policy");
6222        module.drain().await.unwrap();
6223        server.abort();
6224    }
6225
6226    #[cfg(unix)]
6227    #[tokio::test]
6228    async fn http_health_replacement_recovers_after_one_restart() {
6229        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6230        let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6231        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6232        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6233        let mut configured = http_fixture(&dir, &url, 2);
6234        configured.restart.max_restarts = 2;
6235        let module =
6236            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6237                .supervise_configured_with_health(
6238                    configured.module_spec(),
6239                    true,
6240                    configured.health,
6241                    configured.drain_timeout_ms,
6242                    configured.restart,
6243                )
6244                .unwrap();
6245        let first_generation = module.status().unwrap().spawn_generation;
6246        let observed = module.clone();
6247        let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6248        let server = tokio::spawn(async move {
6249            let mut generation = first_generation;
6250            let mut probes = 0;
6251            loop {
6252                let (mut stream, _) = listener.accept().await.unwrap();
6253                let mut request = [0; 2048];
6254                assert!(stream.read(&mut request).await.unwrap() > 0);
6255                let current = observed.status().unwrap().spawn_generation;
6256                if current != generation {
6257                    generation = current;
6258                    probes = 0;
6259                }
6260                probes += 1;
6261                // The original process never recovers. Each replacement has one
6262                // transient miss, then answers successfully for its remaining life.
6263                let healthy = generation != first_generation && probes > 1;
6264                let code = if healthy { 200 } else { 503 };
6265                let response = format!(
6266                    "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6267                );
6268                let _ = stream.write_all(response.as_bytes()).await;
6269                if healthy {
6270                    healthy_tx.send(()).await.unwrap();
6271                }
6272            }
6273        });
6274        timeout(Duration::from_secs(5), async {
6275            let mut healthy_probes = 0;
6276            loop {
6277                if module.state().unwrap() == ModuleState::Failed {
6278                    break;
6279                }
6280                while healthy_rx.try_recv().is_ok() {
6281                    healthy_probes += 1;
6282                }
6283                if healthy_probes >= 3
6284                    && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6285                {
6286                    break;
6287                }
6288                sleep(Duration::from_millis(5)).await;
6289            }
6290        })
6291        .await
6292        .expect("the replacement must settle healthy without exhausting its restart budget");
6293        let status = module.status().unwrap();
6294        module.drain().await.unwrap();
6295        server.abort();
6296        assert_eq!(status.state, ModuleState::Running);
6297        assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6298        assert_eq!(status.health.consecutive_failures, 0);
6299        assert_eq!(status.lifetime_restarts, 1);
6300        assert_eq!(status.restart_count, 1);
6301    }
6302
6303    #[cfg(unix)]
6304    #[tokio::test]
6305    async fn http_health_sustained_failure_exhausts_restart_budget() {
6306        use std::collections::BTreeMap;
6307        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6308        let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6309        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6310        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6311        let mut configured = http_fixture(&dir, &url, 2);
6312        configured.restart.max_restarts = 2;
6313        let module =
6314            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6315                .supervise_configured_with_health(
6316                    configured.module_spec(),
6317                    true,
6318                    configured.health,
6319                    configured.drain_timeout_ms,
6320                    configured.restart,
6321                )
6322                .unwrap();
6323        let observed = module.clone();
6324        let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6325        let served = probes.clone();
6326        let server = tokio::spawn(async move {
6327            loop {
6328                let (mut stream, _) = listener.accept().await.unwrap();
6329                let mut request = [0; 2048];
6330                assert!(stream.read(&mut request).await.unwrap() > 0);
6331                *served
6332                    .lock()
6333                    .unwrap()
6334                    .entry(observed.status().unwrap().spawn_generation)
6335                    .or_default() += 1;
6336                let _ = stream
6337                    .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6338                    .await;
6339            }
6340        });
6341        timeout(Duration::from_secs(5), async {
6342            while module.state().unwrap() != ModuleState::Failed {
6343                sleep(Duration::from_millis(5)).await;
6344            }
6345        })
6346        .await
6347        .expect("sustained failures must still exhaust the health restart budget");
6348        let status = module.status().unwrap();
6349        server.abort();
6350        assert_eq!(status.restart_count, 2);
6351        assert_eq!(status.lifetime_restarts, 2);
6352        assert!(!status.process_alive);
6353        assert_eq!(
6354            probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6355            vec![2, 2, 2]
6356        );
6357    }
6358
6359    #[cfg(unix)]
6360    #[tokio::test]
6361    async fn spawn_clears_the_previous_process_health() {
6362        let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6363        let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6364        configured.health.cadence = Duration::from_secs(3600);
6365        let module =
6366            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6367                .supervise_configured_with_health(
6368                    configured.module_spec(),
6369                    false,
6370                    configured.health,
6371                    configured.drain_timeout_ms,
6372                    configured.restart,
6373                )
6374                .unwrap();
6375        update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6376            state.health.status = SupervisorHealthStatus::Unresponsive;
6377            state.health.consecutive_failures = 3;
6378            state.health.detail = Some("old process refused connections".into());
6379            state.health.metrics = Some(serde_json::json!({"old": true}));
6380            state.health.last_probe_ms = Some(123);
6381            state.health.last_action = Some("restart".into());
6382            state.health.last_action_ms = Some(124);
6383        })
6384        .unwrap();
6385        module.set_enabled(true).await.unwrap();
6386        let health = module.status().unwrap().health;
6387        module.drain().await.unwrap();
6388        assert_eq!(health.consecutive_failures, 0);
6389        assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6390        assert_eq!(health.detail, None);
6391        assert_eq!(health.metrics, None);
6392        assert_eq!(health.last_probe_ms, None);
6393        assert_eq!(health.last_action.as_deref(), Some("restart"));
6394        assert_eq!(health.last_action_ms, Some(124));
6395    }
6396
6397    #[cfg(unix)]
6398    #[tokio::test]
6399    async fn http_health_no_listener_restarts_after_consecutive_failures() {
6400        let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6401        let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6402        let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6403        drop(unused);
6404        let configured = http_fixture(&dir, &url, 2);
6405        let module =
6406            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6407                .supervise_configured_with_health(
6408                    configured.module_spec(),
6409                    true,
6410                    configured.health,
6411                    configured.drain_timeout_ms,
6412                    configured.restart,
6413                )
6414                .unwrap();
6415        let before = module.status().unwrap().spawn_generation;
6416        timeout(Duration::from_secs(5), async {
6417            loop {
6418                let status = module.status().unwrap();
6419                if status.spawn_generation > before {
6420                    assert!(status.lifetime_restarts > 0);
6421                    break;
6422                }
6423                sleep(Duration::from_millis(5)).await;
6424            }
6425        })
6426        .await
6427        .expect("sustained HTTP refusal must trigger the health restart policy");
6428        module.drain().await.unwrap();
6429    }
6430
6431    #[cfg(unix)]
6432    #[tokio::test]
6433    async fn http_health_timeout_honours_deadline() {
6434        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6435        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6436        let server = tokio::spawn(async move {
6437            let _held = listener.accept().await.unwrap();
6438            std::future::pending::<()>().await;
6439        });
6440        let error = timeout(
6441            Duration::from_secs(1),
6442            probe_http_health(&url, Duration::from_millis(10)),
6443        )
6444        .await
6445        .expect("the probe must enforce its own deadline")
6446        .unwrap_err();
6447        server.abort();
6448        assert!(error.to_string().contains("timed out"));
6449    }
6450
6451    #[cfg(unix)]
6452    #[tokio::test]
6453    async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6454        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6455        let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6456        let url = format!(
6457            "http://localhost:{}/healthz",
6458            listener.local_addr().unwrap().port()
6459        );
6460        let server = tokio::spawn(async move {
6461            let (mut stream, _) = listener.accept().await.unwrap();
6462            let mut request = [0u8; 2048];
6463            assert!(stream.read(&mut request).await.unwrap() > 0);
6464            stream
6465                .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6466                .await
6467                .unwrap();
6468        });
6469        // The deadline only bounds a hang. A probe that never tried the IPv6
6470        // address would be refused on 127.0.0.1 and fail at once, so a longer
6471        // deadline does not weaken the assertion; one second timed out under a
6472        // loaded parallel test run.
6473        assert_eq!(
6474            probe_http_health(&url, Duration::from_secs(10))
6475                .await
6476                .unwrap()
6477                .status,
6478            HealthStatus::Ok
6479        );
6480        server.await.unwrap();
6481    }
6482
6483    #[cfg(unix)]
6484    #[tokio::test]
6485    async fn http_health_timeout_keeps_partial_status_and_body() {
6486        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6487        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6488        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6489        let server = tokio::spawn(async move {
6490            let (mut stream, _) = listener.accept().await.unwrap();
6491            let mut request = [0u8; 2048];
6492            assert!(stream.read(&mut request).await.unwrap() > 0);
6493            stream
6494                .write_all(
6495                    b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6496                )
6497                .await
6498                .unwrap();
6499            std::future::pending::<()>().await;
6500        });
6501        let error = probe_http_health(&url, Duration::from_secs(1))
6502            .await
6503            .unwrap_err()
6504            .to_string();
6505        server.abort();
6506        assert!(
6507            error.contains("timed out")
6508                && error.contains("503 Unavailable")
6509                && error.contains("partial diagnostic"),
6510            "{error}"
6511        );
6512    }
6513
6514    #[cfg(unix)]
6515    #[tokio::test]
6516    async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6517        use tokio::io::{AsyncReadExt, AsyncWriteExt};
6518        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6519        let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6520        let server = tokio::spawn(async move {
6521            let (mut stream, _) = listener.accept().await.unwrap();
6522            let mut request = [0u8; 2048];
6523            assert!(stream.read(&mut request).await.unwrap() > 0);
6524            let body = format!("{}not-in-diagnostic", "x".repeat(400));
6525            let response = format!(
6526                "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6527                body.len()
6528            );
6529            stream.write_all(response.as_bytes()).await.unwrap();
6530        });
6531        let error = probe_http_health(&url, Duration::from_secs(1))
6532            .await
6533            .unwrap_err()
6534            .to_string();
6535        server.await.unwrap();
6536        assert!(error.ends_with(&"x".repeat(200)), "{error}");
6537        assert_eq!(error.split(": ").last().unwrap().len(), 200);
6538        assert!(!error.contains("not-in-diagnostic"));
6539    }
6540
6541    #[cfg(unix)]
6542    #[tokio::test]
6543    async fn http_health_real_nats_server_monitoring() {
6544        if std::process::Command::new("nats-server")
6545            .arg("--version")
6546            .env("XDG_DATA_HOME", std::env::temp_dir())
6547            .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6548            .env("XDG_CONFIG_HOME", std::env::temp_dir())
6549            .output()
6550            .is_err()
6551        {
6552            eprintln!(
6553                "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6554            );
6555            return;
6556        }
6557        let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6558        let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6559        let port = monitor.local_addr().unwrap().port();
6560        let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6561        let client_port = client.local_addr().unwrap().port();
6562        let config = dir.join("server.conf");
6563        std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6564        let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6565        configured.program = PathBuf::from("nats-server");
6566        configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6567        drop(monitor);
6568        drop(client);
6569        let module =
6570            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6571                .supervise_configured_with_health(
6572                    configured.module_spec(),
6573                    true,
6574                    configured.health,
6575                    configured.drain_timeout_ms,
6576                    configured.restart,
6577                )
6578                .unwrap();
6579        wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6580        module.drain().await.unwrap();
6581    }
6582
6583    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6584    async fn configuration_update_does_not_replace_captured_running_process_facts() {
6585        let supervisor =
6586            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6587        let snapshot = stale_process_snapshot(ModuleState::Running, true);
6588        let initial = ModuleSpec {
6589            module_id: "rescan-preserves-spawn-facts".to_string(),
6590            program: PathBuf::from("/spawned/module"),
6591            args: Vec::new(),
6592            env: Vec::new(),
6593            reserved: false,
6594            reserved_prefixes: Vec::new(),
6595            protocol: ModuleProtocol::Subc,
6596            overlap: Default::default(),
6597        };
6598        let module = supervisor.supervised_module(
6599            initial.clone(),
6600            supervisor.runtime_config(),
6601            snapshot,
6602            None,
6603        );
6604        let before = module.status().unwrap();
6605        let mut replacement = initial;
6606        replacement.program = PathBuf::from("/rescanned/replacement-module");
6607
6608        module
6609            .update_configuration(replacement, HealthConfig::default(), None)
6610            .await
6611            .unwrap();
6612
6613        let after = module.status().unwrap();
6614        assert_eq!(after.pid, before.pid);
6615        assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6616        assert_eq!(after.spawned_from, before.spawned_from);
6617        drop(module);
6618    }
6619}
6620
6621fn unix_ms_now() -> u64 {
6622    SystemTime::now()
6623        .duration_since(UNIX_EPOCH)
6624        .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6625        .unwrap_or(0)
6626}
6627
6628async fn supervise_loop(
6629    mut spec: ModuleSpec,
6630    mut runtime: SupervisorRuntimeConfig,
6631    registry: Arc<Registry>,
6632    process_liveness: Arc<SupervisorProcessLiveness>,
6633    snapshot: SharedSnapshot,
6634    mut child: Option<SupervisedChild>,
6635    mut commands: mpsc::Receiver<SupervisorCommand>,
6636) {
6637    let mut health_probe = HealthProbeRuntime::default();
6638    // Registry writes (including embedded callers) notify this module only.
6639    // Subscribe before the first refresh; watch retains changes that arrive
6640    // while commands or probes are running. No polling fallback is needed.
6641    let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6642        Ok(changes) => Some(changes),
6643        Err(err) => {
6644            // A poisoned registry cannot accept further writes, so it cannot
6645            // recover via a timer. Keep exit and command handling alive.
6646            warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6647            None
6648        }
6649    };
6650    // All restart backoffs run here, including health and operator requests.
6651    // While one is pending the loop serves commands, so disable or drain can
6652    // cancel the replacement without spawning a process just to stop it.
6653    let mut pending_respawn: Option<PendingRespawn> = None;
6654    // Commands a swap handed back to run next (see `swap::SwapEnd`). Served
6655    // before anything else so a stop that interrupted a swap runs at once.
6656    let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6657    let mut previous_instance_recheck = tokio::time::interval_at(
6658        Instant::now() + Duration::from_secs(5),
6659        Duration::from_secs(5),
6660    );
6661    previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6662    loop {
6663        #[cfg(test)]
6664        {
6665            lock_snapshot(&snapshot).unwrap().actor_select = None;
6666        }
6667        #[cfg(target_os = "macos")]
6668        if let Some(active) = child.as_mut() {
6669            active.confirm_privacy_exec().await;
6670        }
6671        if let Some(scheduled) = runtime
6672            .scheduled_respawn
6673            .lock()
6674            .unwrap_or_else(|p| p.into_inner())
6675            .take()
6676        {
6677            pending_respawn = Some(scheduled);
6678        }
6679        if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6680            pending_respawn = None;
6681            cancel_deferred_reload(
6682                &runtime,
6683                &spec.module_id,
6684                "respawn cancelled by a supervisor command",
6685            );
6686        }
6687        if child.is_none()
6688            && pending_respawn.is_none()
6689            && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6690        {
6691            cancel_deferred_reload(
6692                &runtime,
6693                &spec.module_id,
6694                "respawn cancelled before a replacement was spawned",
6695            );
6696            let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6697                state.respawn_pending = false;
6698                state.coalesced_restart_pending = false;
6699                if matches!(
6700                    state.state,
6701                    ModuleState::Restarting
6702                        | ModuleState::Starting
6703                        | ModuleState::Draining
6704                        | ModuleState::Unresponsive
6705                ) {
6706                    error!(module_id = %spec.module_id, state = ?state.state,
6707                        "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6708                    state.state = ModuleState::Failed;
6709                    clear_current_process_facts(state);
6710                }
6711            });
6712        }
6713        if let Some(command) = requeued.pop_front() {
6714            if !handle_supervisor_command(
6715                command,
6716                &mut spec,
6717                &mut runtime,
6718                &registry,
6719                &process_liveness,
6720                &snapshot,
6721                &mut child,
6722                &mut commands,
6723                &mut requeued,
6724            )
6725            .await
6726            {
6727                return;
6728            }
6729            if child.is_some() || !respawn_still_pending(&snapshot) {
6730                pending_respawn = None;
6731                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6732                    state.respawn_pending = false
6733                });
6734            }
6735            continue;
6736        }
6737        if child.is_some() {
6738            #[cfg(test)]
6739            {
6740                lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6741            }
6742            health_probe.refresh_registration(&spec, &runtime, &registry, &snapshot);
6743            let wake_after = health_probe.wake_after();
6744            let probe_sleep = async move {
6745                match wake_after {
6746                    Some(delay) => sleep(delay).await,
6747                    None => std::future::pending().await,
6748                }
6749            };
6750            tokio::pin!(probe_sleep);
6751            // A `protocol: "none"` child (a plain process that never registers
6752            // over the subc wire, such as nats-server) with no HTTP health
6753            // check has nothing to probe, so it waits only for its exit or a
6754            // supervisor command. A rescan that adds an HTTP check is a command.
6755            let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6756            #[cfg(test)]
6757            {
6758                let registration_pending = wire_child
6759                    && registration_changes
6760                        .as_ref()
6761                        .is_some_and(|changes| changes.has_changed().unwrap());
6762                let mut state = lock_snapshot(&snapshot).unwrap();
6763                state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6764                    generation: state.spawn_generation,
6765                    turn: state.actor_turns,
6766                    registered_connection: health_probe.registered_connection,
6767                    next_probe_at: health_probe.next_probe_at,
6768                    wake_after,
6769                });
6770            }
6771            let active_child = child.as_mut().expect("child checked above");
6772            tokio::select! {
6773                wait_result = active_child.wait() => {
6774                    #[cfg(test)]
6775                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6776                    // Every arm below that gives up on the CHILD must keep the
6777                    // supervision task itself alive (child = None, loop
6778                    // continues into command-serving mode). Returning here
6779                    // closes the command channel, which makes the module
6780                    // permanently unrestartable in-band: a clean child exit
6781                    // of an enabled module once wedged the fleet this way
6782                    // ('supervisor command channel is closed') and required a
6783                    // full daemon restart to recover.
6784                    let exit_report = match wait_result {
6785                        Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6786                        Err(err) => {
6787                            active_child.drain_stderr(&spec.module_id).await;
6788                            fail_after_wait_error(
6789                                &spec.module_id,
6790                                &snapshot,
6791                                &runtime.terminal_ring,
6792                                &runtime.spawn_events,
6793                                format!("failed to wait for supervised module: {err}"),
6794                            );
6795                            untrack_if_registration_released(
6796                                &process_liveness,
6797                                &registry,
6798                                &spec.module_id,
6799                                &snapshot,
6800                            );
6801                            error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6802                            child = None;
6803                            continue;
6804                        }
6805                    };
6806                    active_child.drain_stderr(&spec.module_id).await;
6807
6808                    let next = on_child_exit(
6809                        &spec,
6810                        runtime.restart_policy,
6811                        &registry,
6812                        &snapshot,
6813                        &runtime.terminal_ring,
6814                        &runtime.spawn_events,
6815                        &runtime.child_roster,
6816                        exit_report,
6817                    ).await;
6818                    runtime.start_requester = None;
6819                    runtime.pending_activity_restart = false;
6820                    // The exit is recorded, so a daemon shutdown may stop
6821                    // waiting for this child (see `SupervisedChild::wait`).
6822                    active_child.release_roster();
6823                    match next {
6824                        NextAction::Stop { registration_released } => {
6825                            if registration_released {
6826                                process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6827                            }
6828                            child = None;
6829                        }
6830                        NextAction::Restart { schedule } => {
6831                            let delay = schedule.map_or(
6832                                runtime.restart_policy.delay_for_restart(0),
6833                                |schedule| schedule.delay,
6834                            );
6835                            if let Some(schedule) = schedule {
6836                                log_crash_respawn(&spec.module_id, schedule);
6837                            }
6838                            // The exited child is fully recorded at this point,
6839                            // so release it and count the backoff down in the
6840                            // command-serving branch below rather than sleeping
6841                            // here: commands cannot be received from inside this
6842                            // select arm, and an operator disable or drain that
6843                            // arrives during the backoff must cancel the pending
6844                            // respawn instead of waiting for it to spawn first.
6845                            child = None;
6846                            pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6847                             let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6848                        }
6849                    }
6850                }
6851                command = commands.recv() => {
6852                    #[cfg(test)]
6853                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6854                    let Some(command) = command else {
6855                        return;
6856                    };
6857                    if !handle_supervisor_command(
6858                        command,
6859                        &mut spec,
6860                        &mut runtime,
6861                        &registry,
6862                        &process_liveness,
6863                        &snapshot,
6864                        &mut child,
6865                        &mut commands,
6866                        &mut requeued,
6867                    ).await {
6868                        return;
6869                    }
6870                }
6871                _ = async {
6872                    match registration_changes.as_mut() {
6873                        Some(changes) => { let _ = changes.changed().await; }
6874                        None => std::future::pending().await,
6875                    }
6876                }, if wire_child => {
6877                    #[cfg(test)]
6878                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6879                }
6880                _ = &mut probe_sleep => {
6881                    #[cfg(test)]
6882                    { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6883                    if health_probe.due() {
6884                        run_health_probe_cycle(
6885                            &spec,
6886                            &runtime,
6887                            &registry,
6888                            &process_liveness,
6889                            &snapshot,
6890                            &mut child,
6891                        ).await;
6892                        if child.is_none() {
6893                            runtime.start_requester = None;
6894                            runtime.pending_activity_restart = false;
6895                        }
6896                        if child.is_some() {
6897                            health_probe.schedule_next(&spec, runtime.health.cadence);
6898                        }
6899                    }
6900                }
6901            }
6902        } else if let Some(pending) = pending_respawn {
6903            tokio::select! {
6904                _ = sleep_until(pending.deadline) => {
6905                    pending_respawn = None;
6906                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6907                    // A command handled below while the backoff elapsed may
6908                    // have stopped the module; never respawn past an operator's
6909                    // disable or drain.
6910                    if !respawn_still_pending(&snapshot) {
6911                        continue;
6912                    }
6913                    // The daemon began shutting down during the backoff: the
6914                    // spawn would be refused anyway, and refusing it here
6915                    // leaves the module stopped instead of reporting a
6916                    // failed restart.
6917                    if runtime.child_roster.is_closed() {
6918                        let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6919                            state.state = ModuleState::Stopped;
6920                        });
6921                        debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6922                        continue;
6923                    }
6924                    if let Err(err) = release_dead_registration(
6925                        &registry,
6926                        runtime.forwarding.as_deref(),
6927                        &snapshot,
6928                        &spec.module_id,
6929                    ).await {
6930                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
6931                        error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6932                        continue;
6933                    }
6934
6935                    if matches!(pending.kind, RespawnKind::Reload) {
6936                        let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6937                        let result = finish_reload_child(&spec, &runtime, &registry, &process_liveness, &snapshot, &mut child).await;
6938                        if result.is_ok() && runtime.pending_activity_restart {
6939                            runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6940                        }
6941                        runtime.pending_activity_restart = false;
6942                        if let Some(reply) = reply { let _ = reply.send(result); }
6943                        continue;
6944                    }
6945                    process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6946                    match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6947                        Ok(next_child) => {
6948                            child = Some(next_child);
6949                            if runtime.pending_activity_restart {
6950                                runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6951                                runtime.pending_activity_restart = false;
6952                            }
6953                            debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6954                        }
6955                        Err(err) => {
6956                            fail_snapshot(&snapshot, Some(&spec.module_id), None);
6957                            process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6958                            error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6959                        }
6960                    }
6961                }
6962                command = commands.recv() => {
6963                    let Some(command) = command else {
6964                        return;
6965                    };
6966                    if !handle_supervisor_command(
6967                        command,
6968                        &mut spec,
6969                        &mut runtime,
6970                        &registry,
6971                        &process_liveness,
6972                        &snapshot,
6973                        &mut child,
6974                        &mut commands,
6975                        &mut requeued,
6976                    ).await {
6977                        return;
6978                    }
6979                    // Reconcile the pending respawn with what the command did:
6980                    // a start may already have spawned a fresh child,
6981                    // while a disable or drain moved the snapshot out of the
6982                    // state the respawn was counting down from.
6983                    if child.is_some() || !respawn_still_pending(&snapshot) {
6984                        pending_respawn = None;
6985                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6986                    }
6987                }
6988            }
6989        } else {
6990            let command = if lock_snapshot(&snapshot)
6991                .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6992            {
6993                tokio::select! {
6994                    command = commands.recv() => command,
6995                    _ = previous_instance_recheck.tick() => {
6996                        if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6997                            let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6998                            if enabled && !runtime.child_roster.is_closed() {
6999                                process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7000                                match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7001                                    Ok(next_child) => child = Some(next_child),
7002                                    Err(error) => {
7003                                        fail_snapshot(&snapshot, Some(&spec.module_id), None);
7004                                        error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7005                                    }
7006                                }
7007                            } else {
7008                                let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7009                                    state.waiting_for_previous_pid = None;
7010                                    state.waiting_for_previous_detail = None;
7011                                });
7012                            }
7013                        }
7014                        continue;
7015                    }
7016                }
7017            } else {
7018                commands.recv().await
7019            };
7020            let Some(command) = command else {
7021                return;
7022            };
7023            if !handle_supervisor_command(
7024                command,
7025                &mut spec,
7026                &mut runtime,
7027                &registry,
7028                &process_liveness,
7029                &snapshot,
7030                &mut child,
7031                &mut commands,
7032                &mut requeued,
7033            )
7034            .await
7035            {
7036                return;
7037            }
7038        }
7039    }
7040}
7041
7042fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7043    info!(
7044        module_id,
7045        restart_in_window = schedule.restart_in_window,
7046        delay_ms = schedule.delay.as_millis() as u64,
7047        "respawning after crash"
7048    );
7049}
7050
7051/// Whether the respawn a backoff was counting down to is still wanted. A
7052/// disable or drain handled while the backoff elapsed moves the snapshot out
7053/// of `Restarting`, and the operator's stop must win over the pending respawn,
7054/// so every sleep-then-spawn path re-validates against the live snapshot
7055/// instead of assuming the state it left behind still holds.
7056fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7057    matches!(
7058        lock_snapshot(snapshot),
7059        Ok(state) if state.enabled && state.state == ModuleState::Restarting
7060    )
7061}
7062
7063enum NextAction {
7064    Stop {
7065        registration_released: bool,
7066    },
7067    Restart {
7068        schedule: Option<CrashRestartSchedule>,
7069    },
7070}
7071
7072#[allow(clippy::too_many_arguments)]
7073async fn handle_supervisor_command(
7074    command: SupervisorCommand,
7075    spec: &mut ModuleSpec,
7076    runtime: &mut SupervisorRuntimeConfig,
7077    registry: &Arc<Registry>,
7078    process_liveness: &SupervisorProcessLiveness,
7079    snapshot: &SharedSnapshot,
7080    child: &mut Option<SupervisedChild>,
7081    commands: &mut mpsc::Receiver<SupervisorCommand>,
7082    requeued: &mut VecDeque<SupervisorCommand>,
7083) -> bool {
7084    let command =
7085        if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7086            match command {
7087                SupervisorCommand::Restart { reply, .. }
7088                | SupervisorCommand::Reload { reply, .. }
7089                | SupervisorCommand::Swap { reply, .. } => {
7090                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7091                    return true;
7092                }
7093                SupervisorCommand::SetEnabled {
7094                    enabled: true,
7095                    reply,
7096                    ..
7097                } => {
7098                    let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7099                    return true;
7100                }
7101                other => other,
7102            }
7103        } else {
7104            command
7105        };
7106    match command {
7107        SupervisorCommand::Drain { requester, reply } => {
7108            // A plain stop runs no forwarding drain, so nothing reaches the
7109            // module over its connection before the wait: ask by signal.
7110            let result = drain_optional_child(
7111                &spec.module_id,
7112                spec.protocol,
7113                StopNotice::NotSent,
7114                registry,
7115                runtime.forwarding.as_deref(),
7116                snapshot,
7117                &runtime.terminal_ring,
7118                &runtime.spawn_events,
7119                child,
7120                runtime.drain_timeout,
7121                ModuleState::Stopped,
7122                None,
7123            )
7124            .await;
7125            let registration_released = result.is_ok();
7126            if registration_released {
7127                runtime.record_activity_transition(
7128                    "subc.module_stopped",
7129                    &spec.module_id,
7130                    requester,
7131                );
7132            }
7133            let _ = reply.send(result);
7134            if registration_released {
7135                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7136            }
7137            false
7138        }
7139        SupervisorCommand::Retire { requester, reply } => {
7140            let result = async {
7141                let stop_notice = begin_forwarding_drain_if_configured(
7142                    spec,
7143                    runtime,
7144                    registry,
7145                    snapshot,
7146                    None,
7147                    RouteCloseReason::Disable,
7148                )
7149                .await?;
7150                drain_optional_child(
7151                    &spec.module_id,
7152                    spec.protocol,
7153                    stop_notice,
7154                    registry,
7155                    runtime.forwarding.as_deref(),
7156                    snapshot,
7157                    &runtime.terminal_ring,
7158                    &runtime.spawn_events,
7159                    child,
7160                    runtime.drain_timeout,
7161                    ModuleState::Stopped,
7162                    None,
7163                )
7164                .await
7165            }
7166            .await;
7167            let registration_released = result.is_ok();
7168            if registration_released {
7169                runtime.record_activity_transition(
7170                    "subc.module_stopped",
7171                    &spec.module_id,
7172                    requester,
7173                );
7174            }
7175            let _ = reply.send(result);
7176            if registration_released {
7177                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7178            }
7179            false
7180        }
7181        SupervisorCommand::Restart {
7182            requester,
7183            drain_timeout_ms,
7184            received_at_generation,
7185            queued_at,
7186            reply,
7187        } => {
7188            // Without this line a restart that waited in the queue (behind a
7189            // health probe cycle or another command) was invisible: the log
7190            // showed only the drain timing out, minutes after the operator's call.
7191            info!(
7192                module_id = %spec.module_id,
7193                queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7194                "restart command dequeued"
7195            );
7196            // ACK AT INITIATION, not completion. The blocking form deadlocked any
7197            // caller whose own request lane rides the module being restarted: the
7198            // caller's in-flight request keeps the drain from quiescing, the drain
7199            // keeps the restart from completing, and the completion keeps the reply
7200            // from releasing the caller — so the drain always timed out and cut the
7201            // initiator with a GOODBYE, even on a healthy module. Replying once the
7202            // restart is validated lets a self-lane caller settle, which is exactly
7203            // what makes the drain succeed. Completion is observable via
7204            // supervisor.list / module status; a post-ack failure lands the module
7205            // in a visible terminal state below rather than in a reply nobody can
7206            // receive.
7207            let validation = match lock_snapshot(snapshot) {
7208                Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7209                    module_id: spec.module_id.clone(),
7210                }),
7211                Ok(_) => Ok(()),
7212                Err(err) => Err(err),
7213            };
7214            let initiated = validation.is_ok();
7215            let _ = reply.send(validation);
7216            // A restart asks for a fresh process. Commands run one at a time,
7217            // so a restart queued behind another restart (two operator calls
7218            // in quick succession) is dequeued the moment the first one has
7219            // spawned its replacement -- before that process has sent HELLO.
7220            // Running it would drain and kill the process the first restart
7221            // just produced, which is the opposite of what both callers asked
7222            // for. If a process spawned after this request was received is
7223            // still supervised, the request is already satisfied. Not when the
7224            // configuration changed since that spawn: then the newer process
7225            // predates the spec this restart may exist to apply.
7226            // Generation publication can lag a child that is already serving
7227            // while its identity is recorded. Only process creation after request
7228            // receipt satisfies a restart, not publication after receipt.
7229            let satisfied_by_generation = if initiated
7230                && child
7231                    .as_ref()
7232                    .is_some_and(|child| child.spawned_at > queued_at)
7233            {
7234                lock_snapshot(snapshot).ok().and_then(|state| {
7235                    (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7236                })
7237            } else {
7238                None
7239            };
7240            let satisfied_by_pending = initiated
7241                && child.is_none()
7242                && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7243                    // No child exists yet, so this replacement will necessarily
7244                    // be created after the queued request, unlike a published
7245                    // generation whose process may already have been serving.
7246                    let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7247                    if pending {
7248                        state.coalesced_restart_pending = true;
7249                    }
7250                    pending
7251                });
7252            if satisfied_by_pending {
7253                debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7254            } else if let Some(generation) = satisfied_by_generation {
7255                info!(
7256                    module_id = %spec.module_id,
7257                    received_at_generation,
7258                    "restart already satisfied by generation {generation}; not restarting again"
7259                );
7260            } else if initiated {
7261                // Precedence: this restart's operator override, else the module's
7262                // configured budget (already resolved into the runtime).
7263                let drain_timeout = drain_timeout_ms
7264                    .map(Duration::from_millis)
7265                    .unwrap_or(runtime.drain_timeout);
7266                if let Err(err) = restart_child(
7267                    spec,
7268                    runtime,
7269                    registry,
7270                    process_liveness,
7271                    snapshot,
7272                    child,
7273                    drain_timeout,
7274                )
7275                .await
7276                {
7277                    warn!(
7278                        module_id = %spec.module_id,
7279                        error = %err,
7280                        "operator restart failed after initiation ack; module state carries the outcome"
7281                    );
7282                    let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7283                        state.state = ModuleState::Failed;
7284                        clear_current_process_facts(state);
7285                    });
7286                } else {
7287                    runtime.start_requester = requester;
7288                    runtime.pending_activity_restart = true;
7289                }
7290            }
7291            true
7292        }
7293        SupervisorCommand::Reload { requester, reply } => {
7294            let result =
7295                reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7296            if result.is_ok() {
7297                runtime.start_requester = requester;
7298                runtime.pending_activity_restart = true;
7299            }
7300            if result.is_ok()
7301                && runtime
7302                    .scheduled_respawn
7303                    .lock()
7304                    .unwrap_or_else(|p| p.into_inner())
7305                    .as_ref()
7306                    .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7307            {
7308                *runtime
7309                    .deferred_reload_reply
7310                    .lock()
7311                    .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7312            } else {
7313                let _ = reply.send(result);
7314            }
7315            true
7316        }
7317        SupervisorCommand::SetEnabled {
7318            enabled,
7319            requester,
7320            reply,
7321        } => {
7322            let previous_requester = runtime.start_requester.clone();
7323            if enabled {
7324                runtime.start_requester = requester.clone();
7325            }
7326            let result = set_child_enabled(
7327                spec,
7328                runtime,
7329                registry,
7330                process_liveness,
7331                snapshot,
7332                child,
7333                enabled,
7334            )
7335            .await;
7336            if matches!(result, Ok(true)) {
7337                runtime.pending_activity_restart = false;
7338                if enabled {
7339                    runtime.start_requester = requester;
7340                } else {
7341                    runtime.start_requester = None;
7342                    runtime.record_activity_transition(
7343                        "subc.module_disabled",
7344                        &spec.module_id,
7345                        requester,
7346                    );
7347                }
7348            } else {
7349                runtime.start_requester = previous_requester;
7350            }
7351            let _ = reply.send(result);
7352            true
7353        }
7354        SupervisorCommand::UpdateConfiguration {
7355            spec: next_spec,
7356            health,
7357            drain_timeout_ms,
7358            reply,
7359        } => {
7360            if let Some(handle) = &runtime.supervisor_handle {
7361                handle.apply_identity_configuration(&next_spec);
7362            }
7363            *spec = next_spec;
7364            let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7365                state.configuration_updated_since_spawn = true;
7366            });
7367            let health_changed = runtime.health != health;
7368            runtime.health = health;
7369            // Reset the cadence and old endpoint's failure streak on a live
7370            // health-policy change rather than waiting for its old deadline.
7371            if health_changed {
7372                let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7373                    state.health = ModuleHealthStatus::default();
7374                });
7375            }
7376            runtime.drain_timeout = drain_timeout_ms
7377                .map(Duration::from_millis)
7378                .unwrap_or(runtime.default_drain_timeout);
7379            *runtime
7380                .effective_drain_timeout
7381                .lock()
7382                .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7383            let _ = reply.send(());
7384            true
7385        }
7386        SupervisorCommand::Swap {
7387            requester,
7388            ready_timeout,
7389            reply,
7390        } => {
7391            let mut swap_runtime = runtime.clone();
7392            swap_runtime.start_requester = requester;
7393            let before = lock_snapshot(snapshot)
7394                .ok()
7395                .map(|state| state.spawn_generation);
7396            let end = swap::run_swap(
7397                spec,
7398                &swap_runtime,
7399                registry,
7400                process_liveness,
7401                snapshot,
7402                child,
7403                commands,
7404                ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7405                reply,
7406            )
7407            .await;
7408            if before
7409                != lock_snapshot(snapshot)
7410                    .ok()
7411                    .map(|state| state.spawn_generation)
7412            {
7413                runtime.start_requester = swap_runtime.start_requester;
7414            }
7415            requeued.extend(end.requeue);
7416            true
7417        }
7418    }
7419}
7420
7421async fn restart_child(
7422    spec: &ModuleSpec,
7423    runtime: &SupervisorRuntimeConfig,
7424    registry: &Registry,
7425    process_liveness: &SupervisorProcessLiveness,
7426    snapshot: &SharedSnapshot,
7427    child: &mut Option<SupervisedChild>,
7428    drain_timeout: Duration,
7429) -> Result<(), SuperviseError> {
7430    // Restart cycles a running module; it must not silently start a disabled one.
7431    if !lock_snapshot(snapshot)?.enabled {
7432        return Err(SuperviseError::Disabled {
7433            module_id: spec.module_id.clone(),
7434        });
7435    }
7436    let stop_notice = begin_forwarding_drain_with_timeout(
7437        spec,
7438        runtime,
7439        registry,
7440        snapshot,
7441        None,
7442        RouteCloseReason::Restart,
7443        drain_timeout,
7444    )
7445    .await?;
7446
7447    if child.is_some() {
7448        drain_optional_child(
7449            &spec.module_id,
7450            spec.protocol,
7451            stop_notice,
7452            registry,
7453            runtime.forwarding.as_deref(),
7454            snapshot,
7455            &runtime.terminal_ring,
7456            &runtime.spawn_events,
7457            child,
7458            drain_timeout,
7459            ModuleState::Restarting,
7460            Some(true),
7461        )
7462        .await?;
7463    } else {
7464        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7465            state.enabled = true;
7466            state.state = ModuleState::Restarting;
7467            clear_current_process_facts(state);
7468        })?;
7469        release_dead_registration(
7470            registry,
7471            runtime.forwarding.as_deref(),
7472            snapshot,
7473            &spec.module_id,
7474        )
7475        .await?;
7476    }
7477
7478    reset_restart_count(snapshot, &spec.module_id)?;
7479    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7480    schedule_respawn(
7481        runtime,
7482        snapshot,
7483        &spec.module_id,
7484        runtime.restart_policy.backoff,
7485        RespawnKind::Spawn,
7486    )
7487}
7488
7489async fn reload_child(
7490    spec: &ModuleSpec,
7491    runtime: &SupervisorRuntimeConfig,
7492    registry: &Registry,
7493    process_liveness: &SupervisorProcessLiveness,
7494    snapshot: &SharedSnapshot,
7495    child: &mut Option<SupervisedChild>,
7496) -> Result<(), SuperviseError> {
7497    // Reload cycles a running module; it must not silently start a disabled one.
7498    if !lock_snapshot(snapshot)?.enabled {
7499        return Err(SuperviseError::Disabled {
7500            module_id: spec.module_id.clone(),
7501        });
7502    }
7503    let stop_notice = begin_forwarding_drain(
7504        spec,
7505        runtime,
7506        registry,
7507        snapshot,
7508        Some(true),
7509        RouteCloseReason::Reload,
7510    )
7511    .await?;
7512
7513    if child.is_some() {
7514        drain_optional_child(
7515            &spec.module_id,
7516            spec.protocol,
7517            stop_notice,
7518            registry,
7519            runtime.forwarding.as_deref(),
7520            snapshot,
7521            &runtime.terminal_ring,
7522            &runtime.spawn_events,
7523            child,
7524            runtime.drain_timeout,
7525            ModuleState::Restarting,
7526            Some(true),
7527        )
7528        .await?;
7529    } else {
7530        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7531            state.enabled = true;
7532            state.state = ModuleState::Restarting;
7533            clear_current_process_facts(state);
7534        })?;
7535        release_dead_registration(
7536            registry,
7537            runtime.forwarding.as_deref(),
7538            snapshot,
7539            &spec.module_id,
7540        )
7541        .await?;
7542    }
7543
7544    reset_restart_count(snapshot, &spec.module_id)?;
7545    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7546    schedule_respawn(
7547        runtime,
7548        snapshot,
7549        &spec.module_id,
7550        runtime.restart_policy.backoff,
7551        RespawnKind::Reload,
7552    )
7553}
7554
7555async fn finish_reload_child(
7556    spec: &ModuleSpec,
7557    runtime: &SupervisorRuntimeConfig,
7558    registry: &Registry,
7559    process_liveness: &SupervisorProcessLiveness,
7560    snapshot: &SharedSnapshot,
7561    child: &mut Option<SupervisedChild>,
7562) -> Result<(), SuperviseError> {
7563    process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7564    let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7565        Ok(next_child) => next_child,
7566        Err(err) => {
7567            return handle_reload_spawn_failure(
7568                spec,
7569                runtime,
7570                process_liveness,
7571                snapshot,
7572                child,
7573                format!("new child failed to spawn: {err}"),
7574            )
7575            .await;
7576        }
7577    };
7578    *child = Some(next_child);
7579
7580    let wait_outcome = {
7581        let active_child = child.as_mut().expect("new reload child was just stored");
7582        wait_for_registration_after_reload(
7583            registry,
7584            &spec.module_id,
7585            snapshot,
7586            active_child,
7587            REGISTRY_RELEASE_TIMEOUT,
7588        )
7589        .await?
7590    };
7591
7592    match wait_outcome {
7593        RegistrationWaitOutcome::Registered => {
7594            debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7595            Ok(())
7596        }
7597        RegistrationWaitOutcome::Exited(exit_report) => {
7598            if let Some(active_child) = child.as_mut() {
7599                active_child.drain_stderr(&spec.module_id).await;
7600            }
7601            // Keep the reaped child's roster guard until its terminal is written.
7602            // Shutdown waits on that guard, not on the child Option used for respawn.
7603            let mut exited_child = child.take().expect("exited reload child is still stored");
7604            #[cfg(test)]
7605            if let Some(gate) = &runtime.test_reload_exit_record_gate {
7606                gate.reached.notify_one();
7607                gate.resume.notified().await;
7608            }
7609            let result = handle_reload_child_registration_failure(
7610                spec,
7611                runtime,
7612                registry,
7613                process_liveness,
7614                snapshot,
7615                child,
7616                ReloadRegistrationFailure {
7617                    exit_report: registration_failure_exit_report(exit_report),
7618                    reason: exited_child
7619                        .spawn_failure
7620                        .clone()
7621                        .unwrap_or_else(|| "new child exited before registering".to_string()),
7622                },
7623            )
7624            .await;
7625            exited_child.release_roster();
7626            result
7627        }
7628        RegistrationWaitOutcome::TimedOut => {
7629            let mut timed_out_child = child
7630                .take()
7631                .expect("timed-out reload child is still running");
7632            timed_out_child
7633                .start_kill()
7634                .map_err(|source| SuperviseError::Kill {
7635                    module_id: spec.module_id.clone(),
7636                    source,
7637                })?;
7638            let status = timed_out_child
7639                .wait()
7640                .await
7641                .map_err(|source| SuperviseError::Wait {
7642                    module_id: spec.module_id.clone(),
7643                    source,
7644                })?;
7645            timed_out_child.drain_stderr(&spec.module_id).await;
7646            handle_reload_child_registration_failure(
7647                spec,
7648                runtime,
7649                registry,
7650                process_liveness,
7651                snapshot,
7652                child,
7653                ReloadRegistrationFailure {
7654                    exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7655                        snapshot,
7656                        &timed_out_child,
7657                        &status,
7658                    )),
7659                    reason: format!(
7660                        "new child did not register within {:?}",
7661                        REGISTRY_RELEASE_TIMEOUT
7662                    ),
7663                },
7664            )
7665            .await
7666        }
7667    }
7668}
7669
7670async fn set_child_enabled(
7671    spec: &ModuleSpec,
7672    runtime: &SupervisorRuntimeConfig,
7673    registry: &Registry,
7674    process_liveness: &SupervisorProcessLiveness,
7675    snapshot: &SharedSnapshot,
7676    child: &mut Option<SupervisedChild>,
7677    enabled: bool,
7678) -> Result<bool, SuperviseError> {
7679    let (current_enabled, current_state, respawn_pending) = {
7680        let state = lock_snapshot(snapshot)?;
7681        (state.enabled, state.state, state.respawn_pending)
7682    };
7683    // `start` (enable on an already-enabled module) heals TERMINAL states instead
7684    // of no-op'ing: a module whose restart budget exhausted (Failed) or that exited
7685    // clean (Stopped) has no live process and no other in-band recovery — the
7686    // operator's start is the explicit recovery act and resets the budget. Without
7687    // this arm the only revival was subc-probe --supervisor-restart in a terminal,
7688    // which the 2026-07-14 aft outage proved is a trap when the failed module is
7689    // the one providing every agent's shell.
7690    let revive_terminal = enabled
7691        && current_enabled
7692        && child.is_none()
7693        && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7694            || (current_state == ModuleState::Restarting && !respawn_pending));
7695    if current_enabled == enabled && !revive_terminal {
7696        return Ok(false);
7697    }
7698
7699    if enabled {
7700        update_snapshot(snapshot, Some(&spec.module_id), |state| {
7701            state.enabled = true;
7702            state.state = ModuleState::Starting;
7703            clear_current_process_facts(state);
7704        })?;
7705        #[cfg(test)]
7706        if runtime.test_seed_stale_facts_before_enable_spawn {
7707            update_snapshot(snapshot, Some(&spec.module_id), |state| {
7708                state.process_alive = true;
7709                state.pid = Some(41);
7710                state.spawned_at_ms = Some(42);
7711                state.spawned_from = Some(PathBuf::from("/spawned/module"));
7712                state.spawned_file_identity = Some(SpawnedFileIdentity {
7713                    device: 43,
7714                    inode: 44,
7715                    #[cfg(windows)]
7716                    windows_image: None,
7717                });
7718            })?;
7719        }
7720        release_dead_registration(
7721            registry,
7722            runtime.forwarding.as_deref(),
7723            snapshot,
7724            &spec.module_id,
7725        )
7726        .await?;
7727        reset_restart_count(snapshot, &spec.module_id)?;
7728        process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7729        let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7730            Ok(next_child) => next_child,
7731            Err(err) => {
7732                if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7733                    state.state = ModuleState::Failed;
7734                    clear_current_process_facts(state);
7735                }) {
7736                    error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7737                }
7738                process_liveness.untrack_if_current(&spec.module_id, snapshot);
7739                return Err(err);
7740            }
7741        };
7742        *child = Some(next_child);
7743        debug!(module_id = %spec.module_id, "supervised module enabled");
7744        Ok(true)
7745    } else {
7746        let stop_notice = begin_forwarding_drain_if_configured(
7747            spec,
7748            runtime,
7749            registry,
7750            snapshot,
7751            Some(false),
7752            RouteCloseReason::Disable,
7753        )
7754        .await?;
7755        drain_optional_child(
7756            &spec.module_id,
7757            spec.protocol,
7758            stop_notice,
7759            registry,
7760            runtime.forwarding.as_deref(),
7761            snapshot,
7762            &runtime.terminal_ring,
7763            &runtime.spawn_events,
7764            child,
7765            runtime.drain_timeout,
7766            ModuleState::Disabled,
7767            Some(false),
7768        )
7769        .await?;
7770        debug!(module_id = %spec.module_id, "supervised module disabled");
7771        Ok(true)
7772    }
7773}
7774
7775#[allow(clippy::too_many_arguments)]
7776async fn on_child_exit(
7777    spec: &ModuleSpec,
7778    policy: RestartPolicy,
7779    registry: &Registry,
7780    snapshot: &SharedSnapshot,
7781    terminal_ring: &Arc<Mutex<TerminalRing>>,
7782    spawn_events: &SpawnEventFeed,
7783    roster: &ChildRoster,
7784    exit_report: ExitReport,
7785) -> NextAction {
7786    // Once the daemon has begun shutting down, no exit is a crash to recover
7787    // from: the module is exiting because the daemon is going away (EOF on its
7788    // connection, or a service manager signalling the whole cgroup). Record it
7789    // as such and never schedule a respawn, which would only start a process
7790    // for the shutdown to end again.
7791    if roster.is_closed() {
7792        return on_child_exit_during_daemon_shutdown(
7793            spec,
7794            registry,
7795            snapshot,
7796            terminal_ring,
7797            spawn_events,
7798            exit_report,
7799        )
7800        .await;
7801    }
7802    // Every stop the supervisor itself asks for (operator stop, disable,
7803    // restart, reload, swap, a health restart, a drain that runs out of budget)
7804    // takes the child out of the supervise loop and reaps it in
7805    // `drain_child_to_state`, and daemon shutdown is handled above. So an exit
7806    // that reaches this point was not requested by the daemon.
7807    //
7808    // For a subc-wire module a clean exit is still a stop: those modules are
7809    // written to re-raise SIGTERM, so a stray outside signal already reads as a
7810    // crash, and exiting 0 is a deliberate choice the module made. A
7811    // `protocol: "none"` module is a stock program we cannot change, and many
7812    // of them (nats-server among them) exit 0 on SIGTERM. Treating that as a
7813    // stop would leave the module down for good after any stray signal, so it
7814    // goes through the crash path instead: it spends restart budget, respawns
7815    // with the crash backoff, and ends `failed` when the budget runs out.
7816    let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7817        && running_protocol(spec, snapshot) == ModuleProtocol::None;
7818    match exit_report.kind {
7819        ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7820            info!(
7821                module_id = %spec.module_id,
7822                exit_code = ?exit_report.code,
7823                exit_signal = ?exit_report.signal,
7824                "supervised module exited cleanly"
7825            );
7826            if let Err(err) = publish_after_terminal_record(
7827                &spec.module_id,
7828                || {
7829                    record_terminal(
7830                        &spec.module_id,
7831                        terminal_ring,
7832                        spawn_events,
7833                        &exit_report,
7834                        TerminalDisposition::Stopped,
7835                    )
7836                },
7837                || {
7838                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7839                        state.state = ModuleState::Stopped;
7840                        clear_current_process_facts(state);
7841                        state.last_exit = Some(exit_report.clone());
7842                    })
7843                },
7844            ) {
7845                error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7846            }
7847            let registration_released = match wait_for_registration_release(
7848                registry,
7849                &spec.module_id,
7850                REGISTRY_RELEASE_TIMEOUT,
7851            )
7852            .await
7853            {
7854                Ok(()) => true,
7855                Err(err) => {
7856                    warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7857                    false
7858                }
7859            };
7860            NextAction::Stop {
7861                registration_released,
7862            }
7863        }
7864        ExitKind::Clean | ExitKind::Crash => {
7865            if unrequested_clean_exit_of_protocol_none {
7866                warn!(
7867                    module_id = %spec.module_id,
7868                    exit_code = ?exit_report.code,
7869                    exit_signal = ?exit_report.signal,
7870                    "protocol-none module exited cleanly without a stop request; handling it as a crash"
7871                );
7872            } else {
7873                warn!(
7874                    module_id = %spec.module_id,
7875                    exit_code = ?exit_report.code,
7876                    exit_signal = ?exit_report.signal,
7877                    "supervised module exited abnormally (crash)"
7878                );
7879            }
7880            let mut restart_schedule = None;
7881            let mut disposition = TerminalDisposition::Disabled;
7882            let mut final_state = ModuleState::Disabled;
7883            // Set only when the budget is what stopped the module, so the
7884            // terminal record says which limit was hit rather than leaving
7885            // `failed` to be read as "crashed once, badly".
7886            let mut disposition_detail = lock_snapshot(snapshot)
7887                .ok()
7888                .and_then(|mut state| state.spawn_failure.take());
7889            let now = Instant::now();
7890            if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7891                if state.enabled {
7892                    if let Some(schedule) = state.next_crash_restart(&policy, now) {
7893                        // Reserving the last slot exhausts the budget for the
7894                        // replacement, not for this exit. Publish its scheduled
7895                        // recovery atomically with the reservation so connection
7896                        // cleanup sees it while terminal recording still holds
7897                        // the visible state at Running.
7898                        state.respawn_pending = true;
7899                        final_state = ModuleState::Restarting;
7900                        restart_schedule = Some(schedule);
7901                        disposition = TerminalDisposition::Restarting;
7902                    } else {
7903                        final_state = ModuleState::Failed;
7904                        disposition = TerminalDisposition::Failed;
7905                        let budget = policy.budget_exhausted_detail();
7906                        disposition_detail =
7907                            Some(disposition_detail.take().map_or_else(
7908                                || budget.clone(),
7909                                |cause| format!("{cause}; {budget}"),
7910                            ));
7911                    }
7912                } else {
7913                    disposition = TerminalDisposition::Disabled;
7914                }
7915            }) {
7916                error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7917                return NextAction::Stop {
7918                    registration_released: false,
7919                };
7920            }
7921            if disposition == TerminalDisposition::Failed {
7922                // The window is in the message, not only in the fields: this line
7923                // is read in a scrollback where a bare `max_restarts=3` reads as a
7924                // lifetime cap and sends the operator looking for three crashes
7925                // that never happened together.
7926                error!(
7927                    module_id = %spec.module_id,
7928                    max_restarts = policy.max_restarts,
7929                    window_secs = policy.window.as_secs(),
7930                    "module stopped: {}",
7931                    policy.budget_exhausted_detail()
7932                );
7933            }
7934            if let Err(err) = publish_after_terminal_record(
7935                &spec.module_id,
7936                || {
7937                    record_terminal_with_detail(
7938                        &spec.module_id,
7939                        terminal_ring,
7940                        spawn_events,
7941                        &exit_report,
7942                        disposition,
7943                        disposition_detail,
7944                    )
7945                },
7946                || {
7947                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
7948                        state.state = final_state;
7949                        clear_current_process_facts(state);
7950                        state.last_exit = Some(exit_report.clone());
7951                    })
7952                },
7953            ) {
7954                error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7955            }
7956
7957            if let Some(schedule) = restart_schedule {
7958                NextAction::Restart {
7959                    schedule: Some(schedule),
7960                }
7961            } else {
7962                let registration_released = match wait_for_registration_release(
7963                    registry,
7964                    &spec.module_id,
7965                    REGISTRY_RELEASE_TIMEOUT,
7966                )
7967                .await
7968                {
7969                    Ok(()) => true,
7970                    Err(err) => {
7971                        warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7972                        false
7973                    }
7974                };
7975                NextAction::Stop {
7976                    registration_released,
7977                }
7978            }
7979        }
7980        ExitKind::DeliberateSeverance => {
7981            warn!(
7982                module_id = %spec.module_id,
7983                exit_code = ?exit_report.code,
7984                exit_signal = ?exit_report.signal,
7985                "supervised module exited after deliberate connection severance"
7986            );
7987            let should_restart = match lock_snapshot(snapshot) {
7988                Ok(state) => state.enabled,
7989                Err(err) => {
7990                    error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7991                    return NextAction::Stop {
7992                        registration_released: false,
7993                    };
7994                }
7995            };
7996            let final_state = if should_restart {
7997                ModuleState::Restarting
7998            } else {
7999                ModuleState::Disabled
8000            };
8001            if let Err(err) = publish_after_terminal_record(
8002                &spec.module_id,
8003                || {
8004                    record_terminal(
8005                        &spec.module_id,
8006                        terminal_ring,
8007                        spawn_events,
8008                        &exit_report,
8009                        terminal_disposition(final_state),
8010                    )
8011                },
8012                || {
8013                    update_snapshot(snapshot, Some(&spec.module_id), |state| {
8014                        state.state = final_state;
8015                        clear_current_process_facts(state);
8016                        state.last_exit = Some(exit_report.clone());
8017                        state.lifetime_restarts += 1;
8018                    })
8019                },
8020            ) {
8021                error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8022                return NextAction::Stop {
8023                    registration_released: false,
8024                };
8025            }
8026
8027            if should_restart {
8028                NextAction::Restart { schedule: None }
8029            } else {
8030                let registration_released = match wait_for_registration_release(
8031                    registry,
8032                    &spec.module_id,
8033                    REGISTRY_RELEASE_TIMEOUT,
8034                )
8035                .await
8036                {
8037                    Ok(()) => true,
8038                    Err(err) => {
8039                        warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8040                        false
8041                    }
8042                };
8043                NextAction::Stop {
8044                    registration_released,
8045                }
8046            }
8047        }
8048    }
8049}
8050
8051async fn on_child_exit_during_daemon_shutdown(
8052    spec: &ModuleSpec,
8053    registry: &Registry,
8054    snapshot: &SharedSnapshot,
8055    terminal_ring: &Arc<Mutex<TerminalRing>>,
8056    spawn_events: &SpawnEventFeed,
8057    exit_report: ExitReport,
8058) -> NextAction {
8059    info!(
8060        module_id = %spec.module_id,
8061        exit_code = ?exit_report.code,
8062        exit_signal = ?exit_report.signal,
8063        exit_kind = ?exit_report.kind,
8064        "supervised module exited during daemon shutdown; not restarting it"
8065    );
8066    if let Err(err) = publish_after_terminal_record(
8067        &spec.module_id,
8068        || {
8069            record_terminal(
8070                &spec.module_id,
8071                terminal_ring,
8072                spawn_events,
8073                &exit_report,
8074                TerminalDisposition::DaemonShutdown,
8075            )
8076        },
8077        || {
8078            update_snapshot(snapshot, Some(&spec.module_id), |state| {
8079                state.state = ModuleState::Stopped;
8080                clear_current_process_facts(state);
8081                state.last_exit = Some(exit_report.clone());
8082            })
8083        },
8084    ) {
8085        error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8086    }
8087    let registration_released =
8088        wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8089            .await
8090            .is_ok();
8091    NextAction::Stop {
8092        registration_released,
8093    }
8094}
8095
8096// Keep the status lock out of terminal I/O. Readers may observe the previous
8097// state while recording is in progress, but never a completed exit without its
8098// history. Journal errors are logged by the journal and retained in the ring;
8099// a recording panic still publishes the final state before resuming the unwind.
8100fn publish_after_terminal_record<T>(
8101    module_id: &str,
8102    record: impl FnOnce(),
8103    publish: impl FnOnce() -> T,
8104) -> T {
8105    let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8106    let published = publish();
8107    if let Err(panic) = recorded {
8108        error!(
8109            module_id,
8110            "terminal recording panicked; published final state before resuming unwind"
8111        );
8112        std::panic::resume_unwind(panic);
8113    }
8114    published
8115}
8116
8117fn fail_after_wait_error(
8118    module_id: &str,
8119    snapshot: &SharedSnapshot,
8120    terminal_ring: &Arc<Mutex<TerminalRing>>,
8121    spawn_events: &SpawnEventFeed,
8122    detail: String,
8123) {
8124    let report = wait_error_exit_report();
8125    publish_after_terminal_record(
8126        module_id,
8127        || {
8128            record_terminal_with_detail(
8129                module_id,
8130                terminal_ring,
8131                spawn_events,
8132                &report,
8133                TerminalDisposition::Failed,
8134                Some(detail),
8135            )
8136        },
8137        || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
8138    );
8139}
8140
8141#[cfg(test)]
8142fn record_wait_error_terminal(
8143    module_id: &str,
8144    terminal_ring: &Arc<Mutex<TerminalRing>>,
8145    spawn_events: &SpawnEventFeed,
8146) {
8147    record_terminal(
8148        module_id,
8149        terminal_ring,
8150        spawn_events,
8151        &wait_error_exit_report(),
8152        TerminalDisposition::Failed,
8153    );
8154}
8155
8156fn record_terminal(
8157    module_id: &str,
8158    terminal_ring: &Arc<Mutex<TerminalRing>>,
8159    spawn_events: &SpawnEventFeed,
8160    exit_report: &ExitReport,
8161    disposition: TerminalDisposition,
8162) {
8163    record_terminal_with_detail(
8164        module_id,
8165        terminal_ring,
8166        spawn_events,
8167        exit_report,
8168        disposition,
8169        None,
8170    );
8171}
8172
8173/// The ring lock is held only to capture the read (see
8174/// `TerminalJournal::capture_read`), so this module's exits keep recording
8175/// while the journal files are read. Blocking: it reads files.
8176fn durable_terminal_history_of(
8177    terminal_ring: &Mutex<TerminalRing>,
8178    module_id: &str,
8179) -> subc_control::TerminalHistory {
8180    let read = terminal_ring
8181        .lock()
8182        .unwrap_or_else(|p| p.into_inner())
8183        .capture_durable_history();
8184    read.read(module_id)
8185}
8186
8187fn record_terminal_with_detail(
8188    module_id: &str,
8189    terminal_ring: &Arc<Mutex<TerminalRing>>,
8190    spawn_events: &SpawnEventFeed,
8191    exit_report: &ExitReport,
8192    disposition: TerminalDisposition,
8193    disposition_detail: Option<String>,
8194) {
8195    #[cfg(test)]
8196    terminal_history_tests::before_terminal_record(spawn_events);
8197    spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8198    let record = TerminalRecord {
8199        exit_code: exit_report.code,
8200        exit_signal: exit_report.signal,
8201        at_ms: exit_report.at_ms,
8202        disposition,
8203        exit_kind: exit_report.kind.into(),
8204        disposition_detail,
8205    };
8206    terminal_ring
8207        .lock()
8208        .unwrap_or_else(|poisoned| poisoned.into_inner())
8209        .record_exit(module_id, record);
8210}
8211
8212fn untrack_if_registration_released(
8213    process_liveness: &SupervisorProcessLiveness,
8214    registry: &Registry,
8215    module_id: &str,
8216    snapshot: &SharedSnapshot,
8217) {
8218    match registry.get_module(module_id) {
8219        Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8220        Ok(Some(_)) => {}
8221        Err(err) => {
8222            warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8223        }
8224    }
8225}
8226
8227/// The child's environment plan: inherit the parent's, drop ambient `CK_LOG`,
8228/// then apply the module's configured entries minus daemon-private capture keys.
8229///
8230/// Separated from `spawn_child` only so it can be asserted without spawning a
8231/// process — a duplicate of this logic in a test would pass while the real one
8232/// drifted, which is the defect class this function exists to avoid.
8233/// The subc-wire half of a spawn: `--subc <connection file>` and the launch
8234/// nonce. A `protocol: "none"` module gets neither, because it cannot use
8235/// either and the argument would stop a stock binary from starting at all.
8236/// `SUBC_MODULE_ID` is set on every path since an unread variable is inert.
8237///
8238/// The plain-spawn form, kept for the tests that assert its plan; spawns go
8239/// through [`apply_wire_spawn_args_for_role`].
8240#[cfg(test)]
8241fn apply_wire_spawn_args(
8242    command: &mut Command,
8243    spec: &ModuleSpec,
8244    connection_file_path: Option<&std::path::Path>,
8245    handle: Option<&SupervisorHandle>,
8246) -> Result<Option<NonceHandoff>, SuperviseError> {
8247    apply_wire_spawn_args_for_role(
8248        command,
8249        spec,
8250        connection_file_path,
8251        handle,
8252        SpawnRole::Plain,
8253    )
8254}
8255
8256/// The read end of a spawn's launch-nonce pipe, prepared by
8257/// [`apply_wire_spawn_args_for_role`] and installed as the child's descriptor 3
8258/// by the last pre-exec step, just before `spawn()`. Windows prepares a named
8259/// server before spawn and authenticates its client after the child exists.
8260#[cfg(unix)]
8261type NonceHandoff = subc_os::LaunchNonceHandoff;
8262#[cfg(windows)]
8263type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8264#[cfg(not(any(unix, windows)))]
8265type NonceHandoff = std::convert::Infallible;
8266
8267/// Create the launch nonce that a new module process must present when it
8268/// registers or opens a consumer connection, including a replacement process
8269/// started alongside a still-serving module.
8270///
8271/// A plain spawn replaces the module's recorded nonce. A swap candidate records
8272/// a separate candidate token so the still-serving incumbent and its consumers
8273/// keep their nonce. Both records are installed before the process exists, so
8274/// the child's initial HELLO registration cannot arrive ahead of its nonce.
8275///
8276/// On Unix the nonce is delivered only through a pipe. It is written into
8277/// a pipe whose read end the child gets as descriptor 3, named by
8278/// `SUBC_LAUNCH_NONCE_FD=3:<pipe inode>`: unlike the environment, another
8279/// process of the same user cannot read it with `ps eww`. That handoff is
8280/// returned rather than installed here, because installing it replaces
8281/// whatever the child has at descriptor 3 and so must be the last pre-exec
8282/// step. Later in `spawn_child_in_slot`, `apply_cgroup_placement` first registers
8283/// the Linux callback that writes the process into its cgroup. Only after that
8284/// does `handoff.install_last` register the descriptor-3 replacement, so it
8285/// cannot close a descriptor the cgroup callback still needs.
8286/// Windows keeps SUBC_LAUNCH_NONCE as well until every module reads the named pipe.
8287fn apply_wire_spawn_args_for_role(
8288    command: &mut Command,
8289    spec: &ModuleSpec,
8290    connection_file_path: Option<&std::path::Path>,
8291    handle: Option<&SupervisorHandle>,
8292    role: SpawnRole,
8293) -> Result<Option<NonceHandoff>, SuperviseError> {
8294    command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8295    // SUBC_LAUNCH_NONCE_FD is removed for every child, `protocol: "none"`
8296    // included: a daemon started from a module's process tree inherits it,
8297    // and passing it on would point the child at a descriptor it does not
8298    // have.
8299    command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8300    #[cfg(windows)]
8301    {
8302        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8303        command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8304    }
8305    // Remove inherited or configured copies too: withholding must mean absent.
8306    command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8307    if spec.protocol == ModuleProtocol::None {
8308        return Ok(None);
8309    }
8310    if let Some(connection_file_path) = connection_file_path {
8311        command.arg(SUBC_ARG).arg(connection_file_path);
8312    }
8313
8314    // Every subc-wire spawn receives a fresh one-time launch nonce for consumer
8315    // route.open attestation. Reserved modules additionally use the same nonce
8316    // for HELLO id-squatting protection. A respawn rotates both records.
8317    let nonce = generate_launch_nonce()?;
8318    if let Some(handle) = handle {
8319        match role {
8320            SpawnRole::Plain => {
8321                handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8322                if spec.reserved {
8323                    handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8324                }
8325            }
8326            SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8327        }
8328    }
8329    #[cfg(unix)]
8330    let handoff = {
8331        let handoff =
8332            subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8333                program: spec.program.clone(),
8334                source,
8335                cgroup_path: None,
8336            })?;
8337        command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8338        Some(handoff)
8339    };
8340    #[cfg(windows)]
8341    let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8342    #[cfg(not(any(unix, windows)))]
8343    let handoff = None;
8344    // Older modules still read SUBC_LAUNCH_NONCE to register. Keep that copy
8345    // until the daemon's per-spawn delivery records show that every running
8346    // Windows module has read its secret from the named pipe.
8347    #[cfg(not(unix))]
8348    command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8349    Ok(handoff)
8350}
8351
8352fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8353    command.env_remove(CK_LOG_ENV);
8354    // The spawn role is the supervisor's to set, and only on a swap candidate
8355    // (see `apply_spawn_role`). Removing it here, rather than just not setting
8356    // it, is what makes it absent on a plain spawn: the daemon's own
8357    // environment could carry it, and so could a spec built outside daemon
8358    // config (config refuses it as an `env` key). A module reading it on a
8359    // plain restart would pick the long swap budget and leave callers waiting.
8360    command.env_remove(SUBC_SPAWN_ROLE_ENV);
8361    for (key, value) in &spec.env {
8362        // cortexkit-log currently exposes retention only as a Rust struct, not
8363        // environment names. These values are daemon-private sink metadata and
8364        // must never become a public child-process contract by being inherited.
8365        if matches!(
8366            key.as_str(),
8367            CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8368        ) || key == SUBC_SPAWN_ROLE_ENV
8369        {
8370            continue;
8371        }
8372        command.env(key, value);
8373    }
8374}
8375
8376#[cfg(windows)]
8377fn prepare_windows_nonce_pipe(
8378    command: &mut Command,
8379    spec: &ModuleSpec,
8380    nonce: &str,
8381) -> Option<NonceHandoff> {
8382    match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8383        Ok(handoff) => {
8384            command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8385            // Let a failed pipe read use SUBC_LAUNCH_NONCE so the process can
8386            // still register until every module reads the pipe. A reader without
8387            // this variable set to env returns the pipe error instead of using
8388            // an inherited or configured environment copy.
8389            command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8390            Some(handoff)
8391        }
8392        Err(error) => {
8393            warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8394            None
8395        }
8396    }
8397}
8398
8399#[cfg(windows)]
8400fn start_windows_nonce_pipe(
8401    handoff: Option<NonceHandoff>,
8402    pid: u32,
8403    spec: &ModuleSpec,
8404) -> Option<subc_os::LaunchNoncePipeDelivery> {
8405    // Close an unread pipe after REGISTRY_RELEASE_TIMEOUT (one second), the
8406    // same maximum wait used for registration during a reload. Older modules
8407    // may register using SUBC_LAUNCH_NONCE without ever connecting to the pipe.
8408    handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8409        Ok(delivery) => Some(delivery),
8410        Err(error) => {
8411            warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8412            None
8413        }
8414    })
8415}
8416
8417fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8418    #[cfg(windows)]
8419    if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8420        return Some(
8421            if snapshot
8422                .pipe_consumed
8423                .as_ref()
8424                .is_some_and(|consumed| consumed.get().is_some())
8425            {
8426                "pipe"
8427            } else {
8428                "env"
8429            }
8430            .to_owned(),
8431        );
8432    }
8433    let _ = snapshot;
8434    None
8435}
8436
8437/// Which slot a spawn fills: the module's ordinary one, or the candidate slot
8438/// of a blue/green swap.
8439#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8440enum SpawnRole {
8441    Plain,
8442    SwapCandidate,
8443}
8444
8445/// Set the spawn role for a swap candidate. A plain spawn gets nothing here;
8446/// `apply_child_env` has already removed the variable for every spawn.
8447fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8448    if role == SpawnRole::SwapCandidate {
8449        command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8450    }
8451}
8452
8453fn previous_instance_error(
8454    spec: &ModuleSpec,
8455    pid: u32,
8456    reason: crate::live_children::RetainedReason,
8457) -> SuperviseError {
8458    SuperviseError::Spawn {
8459        program: spec.program.clone(),
8460        source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8461        cgroup_path: None,
8462    }
8463}
8464
8465fn spawn_child(
8466    spec: &ModuleSpec,
8467    connection_file_path: Option<&std::path::Path>,
8468    handle: Option<&SupervisorHandle>,
8469    ring: &Arc<Mutex<StderrRing>>,
8470    capture_logs_dir: Option<&std::path::Path>,
8471    roster: &ChildRoster,
8472    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8473) -> Result<SupervisedChild, SuperviseError> {
8474    spawn_child_in_slot(
8475        spec,
8476        connection_file_path,
8477        handle,
8478        ring,
8479        capture_logs_dir,
8480        roster,
8481        #[cfg(target_os = "linux")]
8482        cgroup_placement,
8483        SpawnRole::Plain,
8484        false,
8485    )
8486}
8487
8488/// Spawn one process of `spec` into a slot.
8489///
8490/// `alternate_slot` picks the process's cgroup label on Linux (see `swap::cgroup_name`).
8491/// A swap candidate needs a different cgroup from the process it is replacing,
8492/// which is still alive: in the same cgroup the two would be one kill domain,
8493/// and killing a failed candidate could take the incumbent with it.
8494///
8495/// The stderr capture file is `<module_id>.stderr.log` for every process of
8496/// the module, whichever slot it is in, because that is the one file
8497/// `ck module logs` reads. During a swap's overlap both processes append to it;
8498/// the daemon writes whole lines, so the two interleave by line, which is also
8499/// the merged view an operator wants while a swap runs.
8500#[allow(clippy::too_many_arguments)]
8501fn spawn_child_in_slot(
8502    spec: &ModuleSpec,
8503    connection_file_path: Option<&std::path::Path>,
8504    handle: Option<&SupervisorHandle>,
8505    ring: &Arc<Mutex<StderrRing>>,
8506    capture_logs_dir: Option<&std::path::Path>,
8507    roster: &ChildRoster,
8508    #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8509    role: SpawnRole,
8510    alternate_slot: bool,
8511) -> Result<SupervisedChild, SuperviseError> {
8512    if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8513        return Err(previous_instance_error(spec, pid, reason));
8514    }
8515    if roster.is_closed() {
8516        return Err(SuperviseError::Spawn {
8517            program: spec.program.clone(),
8518            source: io::Error::other("the daemon is shutting down; not starting a new process"),
8519            cgroup_path: None,
8520        });
8521    }
8522    #[cfg(target_os = "linux")]
8523    let cgroup_name = {
8524        // Slot names alone are not kill domains: a retired incumbent may still
8525        // be draining when a later enable/restart spawns into the same slot.
8526        // Decimal entropy keeps the suffix unambiguous; Placement performs
8527        // the module-id escaping and constructs the filesystem path.
8528        if cgroup_placement.is_none() {
8529            swap::cgroup_name(&spec.module_id, alternate_slot)
8530        } else {
8531            let nonce = generate_launch_nonce()?;
8532            let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8533            // Leave room for byte escaping and the suffix under NAME_MAX. The
8534            // label is only for humans; the nonce identifies the kill domain.
8535            let mut end = spec.module_id.len().min(64);
8536            while !spec.module_id.is_char_boundary(end) {
8537                end -= 1;
8538            }
8539            format!(
8540                "{}_{suffix}",
8541                swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8542            )
8543        }
8544    };
8545    #[cfg(not(target_os = "linux"))]
8546    let _ = alternate_slot;
8547    #[cfg(target_os = "macos")]
8548    let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8549    #[cfg(windows)]
8550    let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8551        program: spec.program.clone(),
8552        source,
8553        cgroup_path: None,
8554    })?;
8555    #[cfg(windows)]
8556    let mut command = Command::new(image_capture.path());
8557    #[cfg(not(any(target_os = "macos", windows)))]
8558    let mut command = Command::new(&spec.program);
8559    command.args(&spec.args);
8560    // AMBIENT `CK_LOG` MUST NOT LEAK INTO AN OTHERWISE UNCONFIGURED MODULE — but
8561    // that is the whole of the intent, so remove that one key rather than the
8562    // environment.
8563    //
8564    // This was `env_clear()` from 0.17.41 until 0.18.3, which achieved the goal
8565    // and took the POSIX environment with it. Modules spawned that way had no
8566    // HOME, XDG_RUNTIME_DIR, TMPDIR or USER, and the consequences ran past
8567    // logging:
8568    //
8569    //   * `connection_file::discover` reads XDG_RUNTIME_DIR and HOME, so with
8570    //     both unset it fell back to the temp dir alone and `ck` could not find
8571    //     a daemon running on the same machine from inside any module's process
8572    //     tree — reporting a path the file has never lived at, which reads as
8573    //     "the daemon did not write its file".
8574    //   * `default_data_home()` with HOME and XDG_DATA_HOME both unset returns
8575    //     the RELATIVE `.local/share`, so a module deriving its own store path
8576    //     resolved it against its own CWD. That is the store-fragmentation
8577    //     defect the daemon already refuses in config (`parse_doc` rejects a
8578    //     relative `storage.data_home`) arriving by derivation instead.
8579    //   * anything a module spawns inherited it: git without ~/.gitconfig,
8580    //     cargo without CARGO_HOME, ssh, python user dirs — all degrading
8581    //     quietly rather than erroring.
8582    //
8583    // Reported by iceteaSA as #104 after deploying 0.18.2, where `ck daemon`
8584    // offered one candidate under /tmp while the file sat in /run/user/1000.
8585    //
8586    // A configured module is unaffected either way: `module_spec()` puts the
8587    // resolved CK_LOG into `spec.env`, which is applied below and therefore
8588    // wins over anything ambient.
8589    apply_child_env(&mut command, spec);
8590    apply_spawn_role(&mut command, role);
8591    let nonce_handoff =
8592        apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8593
8594    #[cfg(target_os = "linux")]
8595    let cgroup_path = cgroup_placement
8596        .map(|placement| placement.module_path(&cgroup_name))
8597        .transpose()
8598        .map_err(|source| SuperviseError::Cgroup {
8599            module_id: spec.module_id.clone(),
8600            source,
8601        })?;
8602    #[cfg(not(target_os = "linux"))]
8603    let cgroup_path: Option<PathBuf> = None;
8604    #[cfg(target_os = "linux")]
8605    if let Some(path) = &cgroup_path {
8606        if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8607            if let Some(placement) = cgroup_placement {
8608                remove_module_cgroup(placement, &cgroup_name);
8609            }
8610            return Err(error);
8611        }
8612    }
8613
8614    let output_sink = if let Some(logs_dir) = capture_logs_dir {
8615        let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8616        match ChildOutputSink::open(&path, capture_retention(spec)) {
8617            Ok(sink) => sink,
8618            Err(error) => {
8619                warn!(
8620                    module_id = %spec.module_id,
8621                    path = %path.display(),
8622                    error = %error,
8623                    "could not open child output capture file; forwarding to stderr"
8624                );
8625                ChildOutputSink::Stderr
8626            }
8627        }
8628    } else {
8629        ChildOutputSink::Stderr
8630    };
8631
8632    command.stdout(Stdio::piped());
8633    command.stderr(Stdio::piped());
8634    command.kill_on_drop(true);
8635    // EACH MODULE LEADS ITS OWN PROCESS GROUP (the child calls setpgid(0, 0)
8636    // before exec). In the daemon's group, a service manager that kills the
8637    // job's process group when the daemon exits (launchd's default) killed
8638    // every module at the same moment its control connection closed, so no
8639    // module ever ran its EOF teardown on a daemon stop. Outside that group a
8640    // module is reached only by the daemon: the EOF it sees when its
8641    // connection closes, and the bounded stop in `child_roster` for anything
8642    // still running after that. On Linux this composes with the cgroup
8643    // placement above: that is a pre_exec write to cgroup.procs, std performs
8644    // setpgid in the child before running pre_exec callbacks, and the two
8645    // change independent process attributes.
8646    //
8647    // stdin is /dev/null because a process outside the terminal's foreground
8648    // group is stopped (SIGTTIN) if it reads the terminal, which a daemon run
8649    // by hand would otherwise hand down. Under a service manager stdin is
8650    // already /dev/null.
8651    #[cfg(unix)]
8652    command.process_group(0);
8653    command.stdin(Stdio::null());
8654    // The LAST pre-exec step, after the cgroup placement above: installing the
8655    // nonce at descriptor 3 replaces whatever the child had there, which could
8656    // be the descriptor an earlier step writes through.
8657    #[cfg(unix)]
8658    if let Some(handoff) = nonce_handoff {
8659        handoff.install_last(command.as_std_mut());
8660    }
8661    #[cfg(not(any(unix, windows)))]
8662    let _ = nonce_handoff;
8663
8664    // Containment, step 1 of 3 (issue #109): create the child suspended so it
8665    // cannot run a single instruction -- and therefore cannot spawn a
8666    // grandchild -- before it is in the job. See `contain_spawned_child` for the
8667    // other two steps and why the window matters.
8668    #[cfg(windows)]
8669    {
8670        subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8671            program: spec.program.clone(),
8672            source,
8673            cgroup_path: None,
8674        })?;
8675        subc_jobobject::suspend_on_create_async(&mut command);
8676    }
8677    // Timestamp before the spawn call: on Unix the child can serve before that
8678    // call even returns. This conservative bound never labels an older child
8679    // as a replacement for a request received during process creation.
8680    let spawned_at = Instant::now();
8681    let mut child = match command.spawn() {
8682        Ok(child) => child,
8683        Err(source) => {
8684            #[cfg(target_os = "linux")]
8685            if let Some(placement) = cgroup_placement {
8686                remove_module_cgroup(placement, &cgroup_name);
8687            }
8688            return Err(SuperviseError::Spawn {
8689                program: spec.program.clone(),
8690                source,
8691                cgroup_path,
8692            });
8693        }
8694    };
8695    #[cfg(test)]
8696    spawn_record_tests::after_create(&child, spec);
8697    // The parent must close its writer now: the acknowledgement pipe reports EOF
8698    // only when every writer is gone, and the child's copy closes when the
8699    // trampoline replaces itself with the module. Command holds only an integer
8700    // in its pre_exec callback, not another writer.
8701    #[cfg(target_os = "macos")]
8702    drop(exec_ack);
8703
8704    // Bind while the child is still suspended, before job assignment resumes it.
8705    // Keeping the executable handle open across CreateProcessW denies rename
8706    // and deletion, so the file cannot be exchanged during spawn. Bind that
8707    // file ID to the process handle returned by spawn, never a reopened PID.
8708    #[cfg(windows)]
8709    let windows_image = match image_capture.bind_tokio(&child) {
8710        Ok(image) => Arc::new(image),
8711        Err(source) => {
8712            kill_and_reap_rejected_child(&mut child);
8713            return Err(SuperviseError::Spawn {
8714                program: spec.program.clone(),
8715                source,
8716                cgroup_path: None,
8717            });
8718        }
8719    };
8720    // Assign the suspended child to its module job; resume only after its
8721    // process identity has been durably recorded for orphan recovery.
8722    #[cfg(windows)]
8723    let job = contain_spawned_child(&mut child, spec)?;
8724    let spawned_at_ms = unix_ms_now();
8725    #[cfg(not(windows))]
8726    let spawned_from = spec.program.clone();
8727    #[cfg(windows)]
8728    let spawned_from = windows_image.path().to_path_buf();
8729    #[cfg(not(windows))]
8730    let spawned_file_identity = spawned_file_identity(&spawned_from);
8731    #[cfg(windows)]
8732    let spawned_file_identity = Some(SpawnedFileIdentity {
8733        device: 0,
8734        inode: 0,
8735        windows_image: Some(Arc::clone(&windows_image)),
8736    });
8737    let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8738        program: spec.program.clone(),
8739        source: io::Error::other("spawned child exposed no live pid"),
8740        cgroup_path: cgroup_path.clone(),
8741    })?;
8742    #[cfg(windows)]
8743    let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8744    #[cfg(not(windows))]
8745    let process_start_time = crate::provenance::process_start_time(pid);
8746    #[cfg(windows)]
8747    let process_start_time = Some(windows_image.start_time());
8748    let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8749    #[cfg(all(test, target_os = "macos"))]
8750    privacy_exec_boundary_tests::before_image_sample(spec, pid);
8751    // Record the committed kernel image, not the configured pathname: PATH
8752    // lookup and shebang interpretation can select a different executable.
8753    // Linux's posix_spawn can return before that image is installed, so the
8754    // observer also waits for the kernel's first-exec flag to clear.
8755    #[cfg(not(windows))]
8756    let recorded_image = observe_spawned_image(pid);
8757    // spawn() confirms only the first exec, into the trampoline. Never persist
8758    // the trampoline image; the asynchronous acknowledgement publishes the
8759    // module image once the trampoline has replaced itself with the module.
8760    #[cfg(target_os = "macos")]
8761    let recorded_image = if privacy_exec.is_some() {
8762        None
8763    } else {
8764        recorded_image
8765    };
8766    #[cfg(target_os = "linux")]
8767    let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8768    #[cfg(not(target_os = "linux"))]
8769    let recorded_cgroup_name = None;
8770    let roster_guard = roster.admit(
8771        spec.module_id.clone(),
8772        pid,
8773        spec.protocol,
8774        process_start_time,
8775        crate::child_roster::RecordedIdentity {
8776            #[cfg(not(windows))]
8777            start_time: recorded_image.map(|image| image.start_time),
8778            #[cfg(windows)]
8779            start_time: process_start_time,
8780            #[cfg(not(windows))]
8781            executable: recorded_image
8782                .and_then(|image| image.executable)
8783                .map(crate::live_children::ExecutableIdentity::from),
8784            #[cfg(windows)]
8785            executable: Some(crate::live_children::ExecutableIdentity::from(
8786                windows_image.identity(),
8787            )),
8788            cgroup_name: recorded_cgroup_name,
8789            #[cfg(target_os = "linux")]
8790            cgroup_placement: cgroup_placement.cloned(),
8791        },
8792    );
8793    let roster_guard = match roster_guard {
8794        Ok(guard) => guard,
8795        Err(source) => {
8796            #[cfg(windows)]
8797            if let Some(job) = &job {
8798                let _ = job.terminate();
8799            }
8800            #[cfg(target_os = "linux")]
8801            kill_module_cgroup(cgroup_placement, &cgroup_name);
8802            kill_and_reap_rejected_child(&mut child);
8803            #[cfg(target_os = "linux")]
8804            if let Some(placement) = cgroup_placement {
8805                remove_module_cgroup(placement, &cgroup_name);
8806            }
8807            return Err(SuperviseError::Spawn {
8808                program: spec.program.clone(),
8809                source: io::Error::other(format!(
8810                    "could not durably record module child: {source}"
8811                )),
8812                cgroup_path,
8813            });
8814        }
8815    };
8816    // The check at the top of this function can pass just before daemon
8817    // shutdown begins, and the process is only in the roster from here on.
8818    // The shutdown stop returns as soon as it finds the roster empty, so a
8819    // process admitted after that look would outlive the daemon. The roster
8820    // is closed before the stop first reads it and admission happens under
8821    // the roster's lock, so either the stop sees this process or this check
8822    // sees the roster closed: end the process now rather than start a module
8823    // the daemon is about to stop.
8824    if roster.is_closed() {
8825        // This child was never admitted, so there is no module protocol shutdown to wait for.
8826        #[cfg(target_os = "linux")]
8827        kill_module_cgroup(cgroup_placement, &cgroup_name);
8828        if let Err(error) = child.start_kill() {
8829            debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8830        }
8831        #[cfg(target_os = "linux")]
8832        if let Some(placement) = cgroup_placement {
8833            // This spawn was never admitted, so shutdown has no roster entry
8834            // to await. Do not detach its cleanup: the runtime could exit
8835            // before that task reaps the rejected child and removes its group.
8836            while matches!(child.try_wait(), Ok(None)) {
8837                std::thread::yield_now();
8838            }
8839            if matches!(
8840                subc_cgroup::kill_module(Some(placement), &cgroup_name),
8841                subc_cgroup::KillOutcome::Killed
8842            ) {
8843                if let Ok(path) = placement.module_path(&cgroup_name) {
8844                    while std::fs::read_to_string(path.join("cgroup.events"))
8845                        .ok()
8846                        .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8847                    {
8848                        std::thread::yield_now();
8849                    }
8850                }
8851            }
8852            remove_module_cgroup(placement, &cgroup_name);
8853        }
8854        drop(roster_guard);
8855        return Err(SuperviseError::Spawn {
8856            program: spec.program.clone(),
8857            source: io::Error::other(
8858                "the daemon began shutting down while this process was starting; ended it",
8859            ),
8860            cgroup_path,
8861        });
8862    }
8863
8864    #[cfg(windows)]
8865    if let Err(error) = resume_suspended_child(pid, spec) {
8866        kill_and_reap_rejected_child(&mut child);
8867        return Err(error);
8868    }
8869
8870    let stdout_pump = match child.stdout.take() {
8871        Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8872        None => {
8873            warn!(
8874                module_id = %spec.module_id,
8875                "spawned child exposed no stdout pipe; file capture will be incomplete"
8876            );
8877            None
8878        }
8879    };
8880    let stderr_pump = match child.stderr.take() {
8881        Some(stderr) => {
8882            let generation = ring
8883                .lock()
8884                .unwrap_or_else(|poisoned| poisoned.into_inner())
8885                .begin_process();
8886            Some(StderrPump {
8887                task: tokio::spawn(pump_stderr_to(
8888                    stderr,
8889                    Arc::clone(ring),
8890                    generation,
8891                    output_sink,
8892                )),
8893                generation,
8894            })
8895        }
8896        None => {
8897            // Spawning succeeded but the pipe did not materialise. Recording it as
8898            // uncaptured keeps the tail honest: the alternative is an empty tail
8899            // that reads as a module which printed nothing.
8900            ring.lock()
8901                .unwrap_or_else(|poisoned| poisoned.into_inner())
8902                .mark_not_captured("stderr pipe was not available on spawn");
8903            warn!(
8904                module_id = %spec.module_id,
8905                "spawned child exposed no stderr pipe; tail will be unavailable"
8906            );
8907            None
8908        }
8909    };
8910
8911    let exec = Arc::new(crate::activity::ExecOutcome::new());
8912    #[cfg(target_os = "macos")]
8913    let pending_exec = privacy_exec.is_some();
8914    #[cfg(not(target_os = "macos"))]
8915    let pending_exec = false;
8916    if !pending_exec {
8917        exec.finish(crate::activity::ExecState::CommittedExec);
8918    }
8919    Ok(SupervisedChild {
8920        child,
8921        exec: crate::activity::ExecOwner(exec),
8922        #[cfg(windows)]
8923        nonce_delivery,
8924        protocol: spec.protocol,
8925        #[cfg(target_os = "linux")]
8926        module_id: cgroup_name,
8927        #[cfg(target_os = "linux")]
8928        cgroup_placement: cgroup_placement.cloned(),
8929        #[cfg(windows)]
8930        job,
8931        stdout_pump,
8932        stderr_pump,
8933        stderr_ring: Arc::clone(ring),
8934        spawned_at,
8935        spawned_at_ms,
8936        spawned_from,
8937        launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8938        spawned_file_identity,
8939        process_start_time,
8940        process_identity,
8941        pid,
8942        roster_guard: Some(roster_guard),
8943        #[cfg(target_os = "macos")]
8944        privacy_exec,
8945        #[cfg(target_os = "macos")]
8946        report_ready: Arc::new(OnceLock::new()),
8947        spawn_failure: None,
8948    })
8949}
8950
8951#[cfg(test)]
8952mod spawn_record_tests {
8953    use super::*;
8954
8955    thread_local! {
8956        static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8957    }
8958
8959    pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8960        LAST_PID.set(child.id());
8961        #[cfg(windows)]
8962        if let Some((_, address)) = _spec
8963            .env
8964            .iter()
8965            .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8966        {
8967            use std::io::{Read, Write};
8968            let mut stream = std::net::TcpStream::connect(address).unwrap();
8969            stream
8970                .write_all(&child.id().unwrap().to_ne_bytes())
8971                .unwrap();
8972            let mut release = [0];
8973            stream.read_exact(&mut release).unwrap();
8974        }
8975    }
8976
8977    #[cfg(windows)]
8978    pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8979        if spec
8980            .env
8981            .iter()
8982            .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8983        {
8984            Err(io::Error::other("injected module job assignment failure"))
8985        } else {
8986            Ok(())
8987        }
8988    }
8989
8990    fn spec() -> ModuleSpec {
8991        ModuleSpec {
8992            module_id: "spawn-record-worker".into(),
8993            #[cfg(unix)]
8994            program: "/bin/sleep".into(),
8995            #[cfg(windows)]
8996            program: std::env::current_exe().unwrap(),
8997            #[cfg(unix)]
8998            args: vec!["60".into()],
8999            #[cfg(windows)]
9000            args: vec![
9001                "--exact".into(),
9002                "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9003                "--nocapture".into(),
9004            ],
9005            env: vec![],
9006            reserved: false,
9007            reserved_prefixes: vec![],
9008            protocol: ModuleProtocol::None,
9009            overlap: Default::default(),
9010        }
9011    }
9012
9013    #[tokio::test]
9014    async fn record_sync_failure_fails_spawn_and_reaps_child() {
9015        let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9016        let supervisor =
9017            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9018                .with_live_children_record(home.join("live-children.json"));
9019        crate::live_children::fail_next_file_sync();
9020        let result = supervisor.spawn(spec());
9021        let failed = result.is_err();
9022        if let Ok(module) = result {
9023            module.stop().await.unwrap();
9024        }
9025        assert!(failed, "record failure must fail spawn");
9026        let pid = LAST_PID.get().expect("the real OS child was created");
9027        assert!(
9028            subc_os::Process::open(pid).unwrap().is_none(),
9029            "child {pid} survived rejected admission or was not reaped"
9030        );
9031    }
9032
9033    #[cfg(target_os = "macos")]
9034    #[test]
9035    #[ignore = "manual on-device latency measurement"]
9036    fn durable_record_write_latency_50() {
9037        let home = cortexkit_test_support::ScratchDir::new("record-latency");
9038        let path = home.join("live-children.json");
9039        let children = vec![crate::live_children::LiveChild {
9040            module_id: "latency-worker".into(),
9041            pid: std::process::id(),
9042            protocol: ModuleProtocol::None,
9043            start_time: Some(1),
9044            executable: None,
9045            cgroup_name: None,
9046        }];
9047        let mut samples = Vec::new();
9048        for _ in 0..50 {
9049            let start = std::time::Instant::now();
9050            crate::live_children::write_record(&path, &children).unwrap();
9051            samples.push(start.elapsed());
9052        }
9053        samples.sort();
9054        eprintln!(
9055            "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9056            (samples[24] + samples[25]) / 2,
9057            samples[47]
9058        );
9059    }
9060
9061    #[cfg(windows)]
9062    #[test]
9063    fn windows_store_touch_fixture() {
9064        if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9065            std::fs::write(path, b"module ran").unwrap();
9066        }
9067    }
9068
9069    #[cfg(windows)]
9070    #[tokio::test]
9071    async fn windows_daemon_fixture() {
9072        let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9073            return;
9074        };
9075        let mut spec = spec();
9076        spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9077        let supervisor =
9078            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9079        let _module = supervisor.spawn(spec).unwrap();
9080    }
9081
9082    #[cfg(windows)]
9083    #[test]
9084    fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9085        use std::io::Read;
9086        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9087        // An outer job models a scheduler/service host. The daemon's root and
9088        // root job must nest without requiring breakaway from this job.
9089        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9090        command
9091            .args([
9092                "--exact",
9093                "supervise::spawn_record_tests::windows_daemon_fixture",
9094                "--nocapture",
9095            ])
9096            .env(
9097                "SUBC_TEST_DAEMON_BARRIER",
9098                listener.local_addr().unwrap().to_string(),
9099            );
9100        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9101        listener.set_nonblocking(true).unwrap();
9102        let deadline = std::time::Instant::now() + Duration::from_secs(30);
9103        let mut stream = loop {
9104            if let Ok((stream, _)) = listener.accept() {
9105                break stream;
9106            }
9107            assert!(
9108                daemon.child.try_wait().unwrap().is_none(),
9109                "daemon failed before rendezvous"
9110            );
9111            assert!(
9112                std::time::Instant::now() < deadline,
9113                "daemon did not reach CreateProcess failpoint"
9114            );
9115            std::thread::yield_now();
9116        };
9117        stream
9118            .set_read_timeout(Some(Duration::from_secs(30)))
9119            .unwrap();
9120        let mut pid = [0; 4];
9121        stream.read_exact(&mut pid).unwrap();
9122        let pid = u32::from_ne_bytes(pid);
9123        assert!(subc_jobobject::process_exists(pid));
9124        // Terminate only the daemon, not its outer job: otherwise the host job
9125        // would also kill a child whose daemon never joined its root job.
9126        daemon.child.kill().unwrap();
9127        daemon.child.wait().unwrap();
9128        let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9129        if !exited {
9130            daemon.job.terminate().unwrap();
9131        }
9132        assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9133    }
9134
9135    #[cfg(windows)]
9136    #[tokio::test]
9137    async fn windows_containment_failure_never_resumes() {
9138        let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9139        let sentinel = home.join("store-touched");
9140        let mut spec = spec();
9141        spec.env
9142            .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9143        spec.env.push((
9144            "SUBC_TEST_STORE_TOUCH".into(),
9145            sentinel.display().to_string(),
9146        ));
9147        let supervisor =
9148            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9149        let result = supervisor.spawn(spec);
9150        // A version that resumes despite assignment failure must have time to
9151        // execute its sentinel before dropping the returned module kills it.
9152        if let Ok(module) = result {
9153            let _ = tokio::time::timeout(Duration::from_secs(10), async {
9154                while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9155                    tokio::task::yield_now().await;
9156                }
9157            })
9158            .await;
9159            let _ = module.stop().await;
9160            panic!(
9161                "assignment failure resumed a module; store touched={}",
9162                sentinel.exists()
9163            );
9164        }
9165        assert!(
9166            !sentinel.exists(),
9167            "refused module executed store-touch sentinel"
9168        );
9169        assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9170    }
9171
9172    #[cfg(windows)]
9173    #[tokio::test]
9174    async fn windows_nested_daemon_fixture() {
9175        let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9176            return;
9177        };
9178        let mut spec = spec();
9179        spec.env.push((
9180            "SUBC_TEST_STORE_TOUCH".into(),
9181            sentinel.to_str().unwrap().into(),
9182        ));
9183        let supervisor =
9184            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9185        let module = supervisor
9186            .spawn(spec)
9187            .expect("per-module job must nest below daemon and host jobs");
9188        tokio::time::timeout(Duration::from_secs(10), async {
9189            while !PathBuf::from(&sentinel).exists() {
9190                tokio::task::yield_now().await;
9191            }
9192        })
9193        .await
9194        .unwrap();
9195        module.stop().await.unwrap();
9196    }
9197
9198    #[cfg(windows)]
9199    #[test]
9200    fn windows_module_job_nests_under_host_job() {
9201        let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9202        let sentinel = home.join("nested-store-touched");
9203        let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9204        command
9205            .args([
9206                "--exact",
9207                "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9208                "--nocapture",
9209            ])
9210            .env("SUBC_TEST_NESTED_STORE", &sentinel);
9211        let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9212        assert!(daemon.child.wait().unwrap().success());
9213        assert!(sentinel.exists(), "nested protocol:none module never ran");
9214    }
9215}
9216
9217#[cfg(target_os = "linux")]
9218pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9219    use subc_cgroup::KillOutcome;
9220    match subc_cgroup::kill_module(placement, module_id) {
9221        KillOutcome::Killed => {}
9222        KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9223            debug!(
9224                module_id,
9225                "cgroup tree kill unavailable; using direct-child kill"
9226            );
9227        }
9228        KillOutcome::IoError { path, error } => {
9229            warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9230        }
9231    }
9232}
9233
9234/// Assign a freshly spawned Windows child without ever resuming on failure.
9235///
9236/// Steps 2 and 3 of the suspended-create contract: the job is created and the
9237/// child assigned **while it is still suspended** (step 1 is
9238/// `suspend_on_create_async` at the spawn site). The caller resumes only after
9239/// the child's roster admission is durable. Failed containment kills and reaps
9240/// the suspended child instead of running an unprotected module.
9241#[cfg(windows)]
9242fn contain_spawned_child(
9243    child: &mut Child,
9244    spec: &ModuleSpec,
9245) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9246    let assignment = || {
9247        let job = subc_jobobject::JobObject::new()?;
9248        #[cfg(test)]
9249        spawn_record_tests::assignment_failpoint(spec)?;
9250        job.assign(child)?;
9251        Ok::<_, io::Error>(job)
9252    };
9253    match assignment() {
9254        Ok(job) => Ok(Some(job)),
9255        Err(source) => {
9256            kill_and_reap_rejected_child(child);
9257            Err(SuperviseError::Spawn {
9258                program: spec.program.clone(),
9259                source: io::Error::other(format!(
9260                    "Windows module job containment failed: {source}"
9261                )),
9262                cgroup_path: None,
9263            })
9264        }
9265    }
9266}
9267
9268/// Rejection is synchronous: do not return a spawn error while the child can
9269/// still run, or rely on kill-on-drop to reap it at some later runtime turn.
9270fn kill_and_reap_rejected_child(child: &mut Child) {
9271    #[cfg(unix)]
9272    if let Some(pid) = child
9273        .id()
9274        .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9275    {
9276        let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9277    }
9278    let _ = child.start_kill();
9279    while matches!(child.try_wait(), Ok(None)) {
9280        std::thread::yield_now();
9281    }
9282}
9283
9284/// Resume a suspended child, killing it if it cannot be started.
9285///
9286/// A suspended process holds a pid and does nothing, so there is no useful
9287/// state to return: the caller gets an error and the spawn fails.
9288#[cfg(windows)]
9289fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9290    if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9291        // Kill it here rather than leaving a suspended process for the caller
9292        // to notice; `kill_on_drop` would eventually do this, but the module
9293        // would have been reported as running in between.
9294        let _ = std::process::Command::new("taskkill.exe")
9295            .args(["/PID", &pid.to_string(), "/T", "/F"])
9296            .stdin(Stdio::null())
9297            .stdout(Stdio::null())
9298            .stderr(Stdio::null())
9299            .status();
9300        return Err(SuperviseError::Spawn {
9301            program: spec.program.clone(),
9302            source,
9303            cgroup_path: None,
9304        });
9305    }
9306    Ok(())
9307}
9308
9309#[cfg(target_os = "linux")]
9310fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9311    match placement.remove_module(module_id) {
9312        Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9313        Err(error) => warn!(
9314            module_id,
9315            error = %error,
9316            "could not remove module cgroup after process exit; continuing teardown"
9317        ),
9318    }
9319}
9320
9321#[cfg(target_os = "linux")]
9322async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9323    // Reaping the direct child is not proof its descendants exited. End the
9324    // residual tree and wait for the kernel's population fact before rmdir;
9325    // otherwise a successful parent wait leaks a directory on each restart.
9326    if matches!(
9327        subc_cgroup::kill_module(Some(placement), module_id),
9328        subc_cgroup::KillOutcome::Killed
9329    ) {
9330        if let Ok(path) = placement.module_path(module_id) {
9331            while std::fs::read_to_string(path.join("cgroup.events"))
9332                .ok()
9333                .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9334            {
9335                sleep(Duration::from_millis(1)).await;
9336            }
9337        }
9338    }
9339    remove_module_cgroup(placement, module_id);
9340}
9341
9342#[cfg(target_os = "linux")]
9343fn apply_cgroup_placement(
9344    command: &mut Command,
9345    spec: &ModuleSpec,
9346    path: &std::path::Path,
9347) -> Result<(), SuperviseError> {
9348    subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9349        module_id: spec.module_id.clone(),
9350        source,
9351    })
9352}
9353
9354fn capture_retention(spec: &ModuleSpec) -> Retention {
9355    let defaults = Retention::default();
9356    let value = |name: &str| {
9357        spec.env
9358            .iter()
9359            .rev()
9360            .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9361    };
9362    Retention {
9363        max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9364            .and_then(|value| value.parse().ok())
9365            .unwrap_or(defaults.max_file_mb),
9366        keep: value(CAPTURE_KEEP_ENV)
9367            .and_then(|value| value.parse().ok())
9368            .unwrap_or(defaults.keep),
9369        max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9370            .and_then(|value| value.parse().ok())
9371            .unwrap_or(defaults.max_age_days),
9372    }
9373}
9374
9375/// A fresh 256-bit CSPRNG launch nonce, lowercase hex. Used to bind a reserved
9376/// module's registration to the exact process the supervisor spawned.
9377fn generate_launch_nonce() -> Result<String, SuperviseError> {
9378    let mut bytes = [0u8; 32];
9379    getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9380        reason: source.to_string(),
9381    })?;
9382    let mut hex = String::with_capacity(64);
9383    for b in bytes {
9384        use std::fmt::Write;
9385        let _ = write!(hex, "{b:02x}");
9386    }
9387    Ok(hex)
9388}
9389
9390/// Constant-time byte comparison so a reserved-nonce mismatch leaks no timing
9391/// signal about how many leading bytes matched.
9392fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9393    if a.len() != b.len() {
9394        return false;
9395    }
9396    let mut diff = 0u8;
9397    for (x, y) in a.iter().zip(b.iter()) {
9398        diff |= x ^ y;
9399    }
9400    diff == 0
9401}
9402
9403/// The kernel's image after an acknowledged exec, shared by ordinary launches
9404/// and privacy trampolines. PATH and shebang interpretation are kernel facts,
9405/// not identities inferred from a configured pathname.
9406#[cfg(not(windows))]
9407fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9408    let process = subc_os::Process::open(pid).ok().flatten()?;
9409    #[cfg(target_os = "linux")]
9410    return sample_after_linux_exec(
9411        || {
9412            let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9413            let committed = linux_exec_committed(&stat);
9414            #[cfg(test)]
9415            spawn_image_tests::report_exec_state(committed);
9416            committed
9417        },
9418        || process.observe(),
9419    );
9420    #[cfg(not(target_os = "linux"))]
9421    process.observe()
9422}
9423
9424/// glibc's posix_spawn parent can wake in exec_mmap before the child's new mm
9425/// is installed. PF_FORKNOEXEC is cleared later in begin_new_exec, after that
9426/// installation. Until it clears, /proc/pid/exe can still name the daemon.
9427#[cfg(target_os = "linux")]
9428fn linux_exec_committed(stat: &str) -> Option<bool> {
9429    const PF_FORKNOEXEC: u64 = 0x40;
9430    let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9431    Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9432}
9433
9434#[cfg(target_os = "linux")]
9435fn sample_after_linux_exec(
9436    mut exec_committed: impl FnMut() -> Option<bool>,
9437    observe: impl FnOnce() -> Option<subc_os::Observation>,
9438) -> Option<subc_os::Observation> {
9439    let deadline = std::time::Instant::now() + Duration::from_secs(1);
9440    while !exec_committed()? {
9441        // A failed or indefinitely stalled exec must not publish the inherited
9442        // image as authority to signal a future orphan. Missing identity keeps
9443        // the existing conservative orphan-sweep behavior.
9444        if std::time::Instant::now() >= deadline {
9445            return None;
9446        }
9447        std::thread::sleep(Duration::from_micros(50));
9448    }
9449    observe()
9450}
9451
9452#[cfg(all(test, target_os = "linux"))]
9453mod spawn_image_tests {
9454    use super::*;
9455    use std::cell::{Cell, RefCell};
9456
9457    enum ExecEvent {
9458        Uncommitted,
9459        Image(Option<subc_os::Observation>),
9460    }
9461
9462    thread_local! {
9463        static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9464    }
9465
9466    pub(super) fn report_exec_state(committed: Option<bool>) {
9467        if committed == Some(false) {
9468            EXEC_EVENTS.with(|events| {
9469                if let Some(events) = events.borrow_mut().take() {
9470                    let _ = events.send(ExecEvent::Uncommitted);
9471                }
9472            });
9473        }
9474    }
9475
9476    #[test]
9477    fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9478        use std::io::{Read, Write};
9479        use std::os::fd::AsRawFd;
9480
9481        let descriptor = std::fs::File::open("/dev/null").unwrap();
9482        let mut command = std::process::Command::new("/bin/sleep");
9483        command.arg("30");
9484        let (mut ready, mut release) =
9485            subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9486                .unwrap();
9487        let spawn = std::thread::spawn(move || command.spawn());
9488        let mut message = [0u8; 8];
9489        ready.read_exact(&mut message).unwrap();
9490        let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9491        let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9492        let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9493        let (events, received) = std::sync::mpsc::channel();
9494        let observer = std::thread::spawn(move || {
9495            EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9496            let image = observe_spawned_image(pid);
9497            let _ = events.send(ExecEvent::Image(image));
9498        });
9499        // Release only once the production observer has seen the uncommitted
9500        // exec, or has incorrectly returned the inherited image. Either way,
9501        // release and reap before asserting so a regression cannot leak a child.
9502        let first = received.recv_timeout(Duration::from_secs(5));
9503        release.write_all(&[1]).unwrap();
9504        let mut child = spawn.join().unwrap().unwrap();
9505        let after = match first {
9506            Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9507                Ok(ExecEvent::Image(image)) => image,
9508                _ => None,
9509            },
9510            Ok(ExecEvent::Image(image)) => image,
9511            Err(_) => None,
9512        };
9513        observer.join().unwrap();
9514        child.kill().unwrap();
9515        child.wait().unwrap();
9516
9517        assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9518        assert_eq!(
9519            inherited.unwrap().executable,
9520            subc_os::file_identity(&std::env::current_exe().unwrap())
9521        );
9522        assert_eq!(
9523            after.unwrap().executable,
9524            subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9525        );
9526    }
9527
9528    #[test]
9529    fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9530        let committed = Cell::new(false);
9531        let inherited = subc_os::Observation {
9532            start_time: 23,
9533            executable: Some(subc_os::FileIdentity {
9534                device: 29,
9535                inode: 31,
9536            }),
9537        };
9538        let module = subc_os::Observation {
9539            start_time: 23,
9540            executable: Some(subc_os::FileIdentity {
9541                device: 29,
9542                inode: 37,
9543            }),
9544        };
9545        let polls = Cell::new(0);
9546        let image = sample_after_linux_exec(
9547            || {
9548                polls.set(polls.get() + 1);
9549                // Hold the inherited image through the first poll, as in the
9550                // posix_spawn parent-wake window, then commit the module image.
9551                committed.set(polls.get() == 2);
9552                Some(committed.get())
9553            },
9554            || Some(if committed.get() { module } else { inherited }),
9555        );
9556        assert_eq!(image, Some(module));
9557        assert_eq!(polls.get(), 2);
9558    }
9559
9560    #[test]
9561    fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9562        assert_eq!(
9563            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9564            Some(false)
9565        );
9566        assert_eq!(
9567            linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9568            Some(true)
9569        );
9570        assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9571        assert_eq!(linux_exec_committed("17 (a) R"), None);
9572    }
9573
9574    #[test]
9575    fn unreadable_exec_state_never_samples_an_inherited_image() {
9576        assert_eq!(
9577            sample_after_linux_exec(
9578                || None,
9579                || panic!("must not sample before exec confirmation")
9580            ),
9581            None
9582        );
9583    }
9584
9585    #[test]
9586    fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9587        assert_eq!(
9588            sample_after_linux_exec(
9589                || Some(false),
9590                || panic!("must not sample after an exec timeout")
9591            ),
9592            None
9593        );
9594    }
9595}
9596
9597/// Resolve once and spawn that exact path, so PATH lookup cannot choose a
9598/// different executable after the no-delete capture was taken.
9599#[cfg(windows)]
9600fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9601    let mut program = spec.program.clone();
9602    if program.is_absolute() || program.components().count() > 1 {
9603        // An explicit path can name a PE image with no extension. Try that
9604        // exact file first, as Command does, before falling back to .exe.
9605        match subc_os::ExecutableCapture::open(&program) {
9606            Ok(capture) => return Ok(capture),
9607            Err(error)
9608                if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9609            Err(error) => return Err(error),
9610        }
9611        program.set_extension("exe");
9612        return subc_os::ExecutableCapture::open(&program);
9613    }
9614    if program.extension().is_none() {
9615        program.set_extension("exe");
9616    }
9617    let custom_path = spec
9618        .env
9619        .iter()
9620        .rev()
9621        .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9622    let mut directories = Vec::new();
9623    // These are Windows' default executable search locations. A configured
9624    // PATH still overrides the inherited PATH, not the system directories.
9625    if let Some(parent) = std::env::current_exe()?.parent() {
9626        directories.push(parent.to_path_buf());
9627    }
9628    directories.push(std::env::current_dir()?);
9629    if let Some(windows) = std::env::var_os("SystemRoot") {
9630        let windows = PathBuf::from(windows);
9631        directories.push(windows.join("System32"));
9632        directories.push(windows.clone());
9633    }
9634    let path = custom_path
9635        .map(|(_, value)| std::ffi::OsString::from(value))
9636        .or_else(|| std::env::var_os("PATH"));
9637    if let Some(path) = path {
9638        directories.extend(std::env::split_paths(&path));
9639    }
9640    for directory in directories {
9641        match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9642            Ok(capture) => return Ok(capture),
9643            Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9644            Err(error) => return Err(error),
9645        }
9646    }
9647    Err(io::Error::new(
9648        io::ErrorKind::NotFound,
9649        "resolved executable not found",
9650    ))
9651}
9652
9653fn spawn_and_mark_running(
9654    spec: &ModuleSpec,
9655    runtime: &SupervisorRuntimeConfig,
9656    snapshot: &SharedSnapshot,
9657) -> Result<SupervisedChild, SuperviseError> {
9658    let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9659    let child = spawn_child(
9660        spec,
9661        runtime.connection_file_path.as_deref(),
9662        runtime.supervisor_handle.as_ref(),
9663        &runtime.stderr_ring,
9664        runtime.capture_logs_dir.as_deref(),
9665        &runtime.child_roster,
9666        #[cfg(target_os = "linux")]
9667        runtime.cgroup_placement.as_ref(),
9668    )?;
9669    set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9670    if let Some(spawn) = activity_spawn {
9671        spawn.complete(snapshot);
9672    }
9673    Ok(child)
9674}
9675
9676enum RegistrationWaitOutcome {
9677    Registered,
9678    Exited(ExitReport),
9679    TimedOut,
9680}
9681
9682struct ReloadRegistrationFailure {
9683    exit_report: ExitReport,
9684    reason: String,
9685}
9686
9687#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9688enum BusyGaugeObservation {
9689    Quiescent,
9690    Busy,
9691    Omitted,
9692}
9693
9694/// What one health answer said about one declared busy gauge.
9695#[derive(Debug, Clone, PartialEq, Eq)]
9696enum GaugeVerdict {
9697    /// The gauge read 0: this gauge is not what holds the drain.
9698    Zero,
9699    /// The gauge read a positive count.
9700    NonZero(u64),
9701    /// The gauge was present but not a non-negative integer (`0.0`, a string,
9702    /// a negative number). It counts as busy, because the daemon cannot tell
9703    /// it is idle. Carries the value as JSON text, cut to a short prefix.
9704    NotACount(String),
9705    /// The health answer had no metrics object, or one without this gauge.
9706    Missing,
9707    /// The probe produced no health answer to read; carries the probe error's
9708    /// evidence label (`no-answer` for a probe that timed out).
9709    ProbeFailed(&'static str),
9710}
9711
9712impl GaugeVerdict {
9713    fn reads_quiet(&self) -> bool {
9714        matches!(self, Self::Zero)
9715    }
9716}
9717
9718impl fmt::Display for GaugeVerdict {
9719    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9720        match self {
9721            Self::Zero => f.write_str("zero"),
9722            Self::NonZero(value) => write!(f, "non_zero({value})"),
9723            Self::NotACount(value) => write!(f, "not_a_count({value})"),
9724            Self::Missing => f.write_str("missing_from_health_body"),
9725            Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9726        }
9727    }
9728}
9729
9730/// The longest JSON text kept for a gauge value that is not a count, so a
9731/// module reporting a large object as a gauge cannot flood the drain log.
9732const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9733
9734/// Read each declared gauge out of one health answer's metrics.
9735fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9736    let metrics = metrics.and_then(Value::as_object);
9737    gauges
9738        .iter()
9739        .map(
9740            |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9741                None => GaugeVerdict::Missing,
9742                Some(value) => match value.as_u64() {
9743                    Some(0) => GaugeVerdict::Zero,
9744                    Some(count) => GaugeVerdict::NonZero(count),
9745                    None => {
9746                        let mut text = value.to_string();
9747                        if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9748                            let mut end = GAUGE_VALUE_TEXT_LIMIT;
9749                            while !text.is_char_boundary(end) {
9750                                end -= 1;
9751                            }
9752                            text.truncate(end);
9753                            text.push_str("...");
9754                        }
9755                        GaugeVerdict::NotACount(text)
9756                    }
9757                },
9758            },
9759        )
9760        .collect()
9761}
9762
9763/// The drain's overall reading of one health answer: omitted when any
9764/// declared gauge is missing, quiescent only when every gauge reads 0.
9765fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9766    if verdicts
9767        .iter()
9768        .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9769    {
9770        BusyGaugeObservation::Omitted
9771    } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9772        BusyGaugeObservation::Quiescent
9773    } else {
9774        BusyGaugeObservation::Busy
9775    }
9776}
9777
9778/// One declared gauge's history over a drain: the latest verdict on it, and
9779/// the latest value actually read for it (which a later failed probe or
9780/// missing gauge does not erase). Times are offsets from the start of the
9781/// drain's wait.
9782#[derive(Debug, Clone, Default, PartialEq, Eq)]
9783struct GaugeTrace {
9784    latest: Option<(GaugeVerdict, Duration)>,
9785    last_value: Option<(GaugeVerdict, Duration)>,
9786}
9787
9788/// What a drain's busy-gauge probes saw, kept so that a drain which runs to
9789/// its ceiling can say in its timeout line which declared gauge held it and
9790/// why. Without it, a gauge that read busy, a gauge the module never
9791/// reported, and a probe that never got an answer all looked the same: a
9792/// drain that ended at its ceiling with nothing in flight.
9793#[derive(Debug, Clone, Default, PartialEq, Eq)]
9794struct DrainGaugeTrace {
9795    /// Health probes the drain sent.
9796    probes: u32,
9797    /// Each declared gauge, in declaration order.
9798    gauges: Vec<(String, GaugeTrace)>,
9799}
9800
9801impl DrainGaugeTrace {
9802    fn new(gauges: &[String]) -> Self {
9803        Self {
9804            probes: 0,
9805            gauges: gauges
9806                .iter()
9807                .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9808                .collect(),
9809        }
9810    }
9811
9812    /// Record one probe's verdicts, one per declared gauge, `at` into the drain.
9813    fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9814        self.probes = self.probes.saturating_add(1);
9815        for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9816            trace.latest = Some((verdict.clone(), at));
9817            if matches!(
9818                verdict,
9819                GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9820            ) {
9821                trace.last_value = Some((verdict.clone(), at));
9822            }
9823        }
9824    }
9825}
9826
9827/// Renders as `gauge: latest verdict @ offset; last value @ offset` for each
9828/// declared gauge, separated by ` | `. A value is shown as its verdict
9829/// (`zero`, `non_zero(n)`, `not_a_count(json)`), so it says both what was read
9830/// and whether that counted as busy.
9831impl fmt::Display for DrainGaugeTrace {
9832    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9833        if self.gauges.is_empty() {
9834            return f.write_str("none declared");
9835        }
9836        for (index, (name, trace)) in self.gauges.iter().enumerate() {
9837            if index > 0 {
9838                f.write_str(" | ")?;
9839            }
9840            write!(f, "{name}: ")?;
9841            match &trace.latest {
9842                Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9843                None => f.write_str("never probed")?,
9844            }
9845            match &trace.last_value {
9846                Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9847                None => f.write_str("; no value ever read")?,
9848            }
9849        }
9850        Ok(())
9851    }
9852}
9853
9854pub(crate) fn declared_busy_gauges(
9855    registry: &Registry,
9856    module_id: &str,
9857) -> Result<Vec<String>, SuperviseError> {
9858    busy_gauges_of(
9859        registry
9860            .get_module(module_id)
9861            .map_err(SuperviseError::Registry)?,
9862    )
9863}
9864
9865/// [`declared_busy_gauges`] for the registration a connection holds, in any
9866/// slot: after cutover the incumbent is no longer the id's active
9867/// registration, and its own manifest is the one that names its gauges.
9868fn declared_busy_gauges_for_connection(
9869    registry: &Registry,
9870    connection_id: ConnectionId,
9871) -> Result<Vec<String>, SuperviseError> {
9872    busy_gauges_of(
9873        registry
9874            .get_module_by_connection(connection_id)
9875            .map_err(SuperviseError::Registry)?,
9876    )
9877}
9878
9879fn busy_gauges_of(
9880    registration: Option<crate::registry::ModuleRegistration>,
9881) -> Result<Vec<String>, SuperviseError> {
9882    let Some(registration) = registration else {
9883        return Ok(Vec::new());
9884    };
9885    let Some(self_signals) = registration.manifest.self_signals else {
9886        return Ok(Vec::new());
9887    };
9888
9889    let mut gauges = Vec::new();
9890    for declaration in self_signals {
9891        if declaration.kind != SelfSignalKind::Busy {
9892            continue;
9893        }
9894        match declaration.anchored_to {
9895            SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9896                gauges.extend(declared)
9897            }
9898            _ => {
9899                // An invalid Busy anchor is fail-safe: the empty name cannot be
9900                // present in a conforming health report, so this drain stays busy.
9901                gauges.push(String::new());
9902            }
9903        }
9904    }
9905    Ok(gauges)
9906}
9907
9908/// Wait for `endpoint` to have nothing in flight and, when the module declares
9909/// busy gauges, for a health probe to report them quiet. The probe is addressed
9910/// by `scope`: a swap's superseded incumbent must be asked about its own
9911/// gauges, and by module id the probe would reach the promoted candidate.
9912/// While the gauges read busy, the probe repeats every
9913/// [`DRAIN_BUSY_REPROBE_INTERVAL`] until they read quiet or `deadline` passes.
9914/// Every probe's reading of each gauge is recorded in `trace`, for the caller's
9915/// timeout line.
9916#[allow(clippy::too_many_arguments)]
9917async fn wait_for_forwarding_quiescence(
9918    forwarding: &ForwardingTable,
9919    module_id: &str,
9920    runtime: &SupervisorRuntimeConfig,
9921    endpoint: crate::ModuleEndpointId,
9922    deadline: Instant,
9923    busy_gauges: &[String],
9924    scope: DrainScope,
9925    trace: &mut DrainGaugeTrace,
9926) -> Result<bool, SuperviseError> {
9927    let wait_started = Instant::now();
9928    let mut gauges_quiescent = busy_gauges.is_empty();
9929    let mut next_probe_at = wait_started;
9930    let mut omission_reported = false;
9931
9932    loop {
9933        let now = Instant::now();
9934        if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9935            let report = match scope {
9936                DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9937                DrainScope::Endpoint(endpoint) => {
9938                    probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9939                }
9940            };
9941            let verdicts = match &report {
9942                Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9943                Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9944            };
9945            trace.record(&verdicts, wait_started.elapsed());
9946            gauges_quiescent = match report {
9947                Ok(_) => match busy_gauge_observation(&verdicts) {
9948                    BusyGaugeObservation::Quiescent => true,
9949                    BusyGaugeObservation::Busy => false,
9950                    BusyGaugeObservation::Omitted => {
9951                        // Once per drain: a module that never reports a gauge
9952                        // it declared would otherwise say so on every probe.
9953                        if !omission_reported {
9954                            forwarding
9955                                .counters()
9956                                .increment_drains_with_undeclared_gauge();
9957                            let missing = busy_gauges
9958                                .iter()
9959                                .zip(&verdicts)
9960                                .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9961                                .map(|(gauge, _)| gauge.as_str())
9962                                .collect::<Vec<_>>()
9963                                .join(",");
9964                            warn!(
9965                                module_id,
9966                                missing_gauges = %missing,
9967                                "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9968                            );
9969                            omission_reported = true;
9970                        }
9971                        false
9972                    }
9973                },
9974                Err(err) => {
9975                    warn!(
9976                        module_id,
9977                        error = %err,
9978                        "drain health.check did not produce declared busy gauges; treating module as busy"
9979                    );
9980                    false
9981                }
9982            };
9983            // Counted from the probe's completion: the probe above was awaited,
9984            // so no second probe can start while one is still outstanding.
9985            next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9986        }
9987
9988        let in_flight = forwarding
9989            .endpoint_in_flight_count(endpoint)
9990            .map_err(SuperviseError::Forwarding)?;
9991        if in_flight == 0 && gauges_quiescent {
9992            return Ok(true);
9993        }
9994
9995        let now = Instant::now();
9996        if now >= deadline {
9997            return Ok(false);
9998        }
9999        let mut wait = deadline
10000            .saturating_duration_since(now)
10001            .min(REGISTRY_RELEASE_POLL);
10002        if !busy_gauges.is_empty() {
10003            wait = wait.min(next_probe_at.saturating_duration_since(now));
10004        }
10005        sleep(wait).await;
10006    }
10007}
10008
10009/// The wait between drain busy-gauge probes: the larger of
10010/// [`DRAIN_BUSY_REPROBE_INTERVAL`] and [`REGISTRY_RELEASE_POLL`], the
10011/// drain's own in-flight poll interval.
10012fn drain_busy_reprobe_interval() -> Duration {
10013    DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10014}
10015
10016/// Logs why a drain gave up: the drain waited its full timeout without the
10017/// module going quiet, and teardown is being forced.
10018///
10019/// The line names two kinds of evidence, because a bare "timed out" hides
10020/// causes that need different fixes:
10021/// - `holdouts`: the requests and routes still open when the timeout hit,
10022///   and on which connections. One stuck call is a module bug; a held stream
10023///   on every session is a module that should end its streams when it
10024///   receives `module.draining`.
10025/// - `gauges`: for each busy gauge the module declares, its latest reading
10026///   and how many probes ran. This separates a module that kept reporting
10027///   itself busy, one whose health answer never contained the gauge, and one
10028///   that never answered the probes, which otherwise all look like a timeout
10029///   with nothing in flight.
10030fn warn_drain_timed_out(
10031    module_id: &str,
10032    drain_timeout: Duration,
10033    reason: RouteCloseReason,
10034    holdouts: &crate::forwarding::DrainHoldouts,
10035    gauges: &DrainGaugeTrace,
10036) {
10037    warn!(
10038        module_id = %module_id,
10039        waited = ?drain_timeout,
10040        ?reason,
10041        held_requests = holdouts.requests,
10042        held_routes = holdouts.routes,
10043        total_routes = holdouts.total_routes,
10044        top_connections = ?holdouts.top_connections,
10045        // `module_channel:corr`, so the module can find each held request
10046        // in its own log; capped, so `held_requests` is the full count.
10047        held = %holdouts
10048            .held
10049            .iter()
10050            .map(|(channel, corr)| format!("{channel}:{corr}"))
10051            .collect::<Vec<_>>()
10052            .join(","),
10053        gauge_probes = gauges.probes,
10054        busy_gauges = %gauges,
10055        "route drain timed out before request quiescence; forcing teardown"
10056    );
10057}
10058
10059/// The `route.closed` `drained` value implied by a quiescence-wait outcome.
10060///
10061/// `Ok` is always honest and passed straight through -- the wait actually measured
10062/// in-flight state. `Err` means the wait produced no measurement at all (the
10063/// forwarding table's lock was poisoned), so `false` is reported as the one honest
10064/// constant: the drain did not complete. Never recomputed from route state, never a
10065/// third "unknown" value -- the caller must still send a well-formed `route.closed`.
10066fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10067    match wait_result {
10068        Ok(drained) => *drained,
10069        Err(_) => false,
10070    }
10071}
10072
10073fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10074    for released in released_routes {
10075        let frame = match Frame::build_with_version(
10076            released.negotiated_ver,
10077            FrameType::Goodbye,
10078            control_flags(),
10079            released.channel,
10080            released.epoch,
10081            0,
10082            Vec::new(),
10083        ) {
10084            Ok(frame) => frame,
10085            Err(err) => {
10086                warn!(
10087                    route_channel = released.channel,
10088                    error = %err,
10089                    "failed to build supervisor drain route GOODBYE frame"
10090                );
10091                continue;
10092            }
10093        };
10094        if !released.close_on_delivery_failure() {
10095            crate::forwarding::send_module_route_goodbye(
10096                &forwarding.counters(),
10097                &released.sink,
10098                frame,
10099                released.module_id.as_deref(),
10100                "supervisor drain",
10101            );
10102            continue;
10103        }
10104        if let Err(err) = released.sink.try_send(frame) {
10105            warn!(
10106                target_connection_id = released.connection_id.get(),
10107                route_channel = released.channel,
10108                error = %err,
10109                "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10110            );
10111            let _ = forwarding.escalate_client_delivery_failure(
10112                released.connection_id,
10113                released.channel,
10114                released.epoch,
10115                CloseReason::new(
10116                    "route_goodbye_delivery_failed",
10117                    format!(
10118                        "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10119                        released.channel
10120                    ),
10121                ),
10122                crate::forwarding::UndeliveredFrame {
10123                    module_id: released.module_id.as_deref(),
10124                    sink: &released.sink,
10125                },
10126            );
10127        }
10128    }
10129}
10130
10131fn send_module_draining(
10132    module_id: &str,
10133    reason: RouteCloseReason,
10134    deadline_ms: u64,
10135    target: &ModuleDrainTarget,
10136) {
10137    let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10138        reason,
10139        deadline_ms,
10140    }) {
10141        Ok(body) => body,
10142        Err(err) => {
10143            warn!(
10144                module_id,
10145                error = %err,
10146                "failed to encode module draining command"
10147            );
10148            return;
10149        }
10150    };
10151    let frame = match Frame::build_with_version(
10152        target.negotiated_ver,
10153        FrameType::Push,
10154        control_flags(),
10155        0,
10156        0,
10157        0,
10158        body,
10159    ) {
10160        Ok(frame) => frame,
10161        Err(err) => {
10162            warn!(
10163                module_id,
10164                error = %err,
10165                "failed to build module draining command frame"
10166            );
10167            return;
10168        }
10169    };
10170    if let Err(err) = target.sink.try_send(frame) {
10171        warn!(
10172            module_id,
10173            target_connection_id = target.endpoint.connection_id.get(),
10174            error = %err,
10175            "module draining command was not delivered to peer"
10176        );
10177    }
10178}
10179
10180/// The channel-0 GOODBYE that tells a module its stop is planned.
10181fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10182    match Frame::build_with_version(
10183        negotiated_ver,
10184        FrameType::Goodbye,
10185        control_flags(),
10186        0,
10187        0,
10188        0,
10189        Vec::new(),
10190    ) {
10191        Ok(frame) => Some(frame),
10192        Err(err) => {
10193            warn!(
10194                module_id,
10195                error = %err,
10196                "failed to build module GOODBYE frame"
10197            );
10198            None
10199        }
10200    }
10201}
10202
10203/// Send every registered module connection its module GOODBYE at daemon
10204/// shutdown, then request that connection's close.
10205///
10206/// A module tells a planned stop from a lost daemon by whether a GOODBYE came
10207/// before EOF, so the GOODBYE must reach the socket before the close. A close
10208/// request does not wait for the connection's queued frames: its writer gets a
10209/// bounded grace after the close, is aborted if it overruns it, and the daemon
10210/// process may exit before that grace ends. So with `wait_for_flush`, each
10211/// connection is closed only after its writer has acknowledged writing the
10212/// GOODBYE, or once a short shared budget runs out, so one module that is not
10213/// reading cannot hold up the others or the shutdown. Without it the GOODBYEs
10214/// are only queued, for a shutdown the operator has told to stop waiting.
10215/// A connection that is already gone is skipped.
10216#[cfg(unix)]
10217async fn send_module_goodbyes_for_daemon_shutdown(
10218    forwarding: &Arc<ForwardingTable>,
10219    reason: &CloseReason,
10220    wait_for_flush: bool,
10221) {
10222    const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10223    let targets = match forwarding.module_connections() {
10224        Ok(targets) => targets,
10225        Err(err) => {
10226            warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10227            return;
10228        }
10229    };
10230    let deadline = Instant::now() + GOODBYE_BUDGET;
10231    let mut sends = tokio::task::JoinSet::new();
10232    for target in targets {
10233        let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10234            continue;
10235        };
10236        if !wait_for_flush {
10237            if let Err(err) = target.sink.try_send(frame) {
10238                debug!(
10239                    module_id = %target.module_id,
10240                    error = %err,
10241                    "shutdown module GOODBYE was not queued"
10242                );
10243            }
10244            continue;
10245        }
10246        let forwarding = Arc::clone(forwarding);
10247        let reason = reason.clone();
10248        sends.spawn(async move {
10249            match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10250                Ok(Ok(())) => {}
10251                Ok(Err(err)) => debug!(
10252                    module_id = %target.module_id,
10253                    error = %err,
10254                    "module connection closed before its shutdown GOODBYE was written"
10255                ),
10256                Err(_) => warn!(
10257                    module_id = %target.module_id,
10258                    budget = ?GOODBYE_BUDGET,
10259                    "shutdown module GOODBYE was not written within its budget; closing anyway"
10260                ),
10261            }
10262            forwarding.request_connection_close(target.endpoint.connection_id, reason);
10263        });
10264    }
10265    // Every task ends by the shared deadline, so this wait is bounded too.
10266    while sends.join_next().await.is_some() {}
10267}
10268
10269fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10270    let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10271        return;
10272    };
10273    if let Err(err) = target.sink.try_send(frame) {
10274        warn!(
10275            module_id,
10276            target_connection_id = target.endpoint.connection_id.get(),
10277            error = %err,
10278            "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10279        );
10280        forwarding.request_connection_close(
10281            target.endpoint.connection_id,
10282            CloseReason::new(
10283                "module_goodbye_delivery_failed",
10284                format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10285            ),
10286        );
10287    }
10288}
10289
10290#[derive(Clone, Copy)]
10291struct ForwardingDrainContext<'a> {
10292    spec: &'a ModuleSpec,
10293    runtime: &'a SupervisorRuntimeConfig,
10294    registry: &'a Registry,
10295    scope: DrainScope,
10296}
10297
10298/// Which process a forwarding drain addresses.
10299#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10300enum DrainScope {
10301    /// Whatever endpoint is active for the module id: every plain stop,
10302    /// restart and reload. Also moves the module's state to `Draining`.
10303    Active,
10304    /// One specific endpoint: a swap's incumbent after cutover. Draining it by
10305    /// module id would resolve to the promoted candidate and leave neither
10306    /// process routable. The module's state is left alone, since the promoted
10307    /// candidate is what it describes and that process is running.
10308    Endpoint(crate::ModuleEndpointId),
10309}
10310
10311/// Whether a child being drained has already been asked to stop by the time
10312/// its drain wait starts.
10313///
10314/// The drain wait is the same budget whatever this says. What it decides is
10315/// whether the supervisor must ask by signal before that wait begins: a child
10316/// that nobody asked will sit out the whole budget and then be SIGKILLed,
10317/// healthy or not.
10318#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10319enum StopNotice {
10320    /// The module was sent `module.draining` and a module GOODBYE over its own
10321    /// registered connection, and stops itself.
10322    SentOverConnection,
10323    /// The forwarding drain found no registered connection for the module: a
10324    /// subc child spawned moments ago that has not sent HELLO yet, or a
10325    /// `protocol: "none"` child, which never registers.
10326    NoConnection,
10327    /// This path sends nothing over the module's connection: the supervisor has
10328    /// no forwarding table, or the caller stops the child without a forwarding
10329    /// drain.
10330    NotSent,
10331}
10332
10333async fn begin_forwarding_drain(
10334    spec: &ModuleSpec,
10335    runtime: &SupervisorRuntimeConfig,
10336    registry: &Registry,
10337    snapshot: &SharedSnapshot,
10338    enabled: Option<bool>,
10339    reason: RouteCloseReason,
10340) -> Result<StopNotice, SuperviseError> {
10341    let Some(forwarding) = runtime.forwarding.as_ref() else {
10342        return Err(SuperviseError::ReloadUnavailable {
10343            module_id: spec.module_id.clone(),
10344            reason: "supervisor was not configured with a forwarding table".to_string(),
10345        });
10346    };
10347
10348    begin_forwarding_drain_with(
10349        forwarding,
10350        ForwardingDrainContext {
10351            spec,
10352            runtime,
10353            registry,
10354            scope: DrainScope::Active,
10355        },
10356        snapshot,
10357        enabled,
10358        reason,
10359        runtime.drain_timeout,
10360    )
10361    .await
10362}
10363
10364async fn begin_forwarding_drain_if_configured(
10365    spec: &ModuleSpec,
10366    runtime: &SupervisorRuntimeConfig,
10367    registry: &Registry,
10368    snapshot: &SharedSnapshot,
10369    enabled: Option<bool>,
10370    reason: RouteCloseReason,
10371) -> Result<StopNotice, SuperviseError> {
10372    begin_forwarding_drain_with_timeout(
10373        spec,
10374        runtime,
10375        registry,
10376        snapshot,
10377        enabled,
10378        reason,
10379        runtime.drain_timeout,
10380    )
10381    .await
10382}
10383
10384/// Like [`begin_forwarding_drain_if_configured`] but with an explicit drain
10385/// budget, for paths where the operator overrides the module's configured one
10386/// (`supervisor.restart{drain_timeout_ms}`).
10387async fn begin_forwarding_drain_with_timeout(
10388    spec: &ModuleSpec,
10389    runtime: &SupervisorRuntimeConfig,
10390    registry: &Registry,
10391    snapshot: &SharedSnapshot,
10392    enabled: Option<bool>,
10393    reason: RouteCloseReason,
10394    drain_timeout: Duration,
10395) -> Result<StopNotice, SuperviseError> {
10396    let Some(forwarding) = runtime.forwarding.as_ref() else {
10397        return Ok(StopNotice::NotSent);
10398    };
10399
10400    begin_forwarding_drain_with(
10401        forwarding,
10402        ForwardingDrainContext {
10403            spec,
10404            runtime,
10405            registry,
10406            scope: DrainScope::Active,
10407        },
10408        snapshot,
10409        enabled,
10410        reason,
10411        drain_timeout,
10412    )
10413    .await
10414}
10415
10416async fn begin_forwarding_drain_with(
10417    forwarding: &ForwardingTable,
10418    context: ForwardingDrainContext<'_>,
10419    snapshot: &SharedSnapshot,
10420    enabled: Option<bool>,
10421    reason: RouteCloseReason,
10422    drain_timeout: Duration,
10423) -> Result<StopNotice, SuperviseError> {
10424    let ForwardingDrainContext {
10425        spec,
10426        runtime,
10427        registry,
10428        scope,
10429    } = context;
10430    debug_assert_ne!(reason, RouteCloseReason::Crash);
10431    let terminal = matches!(reason, RouteCloseReason::Disable);
10432    let drain_started_at = Instant::now();
10433    let drain_deadline = drain_started_at + drain_timeout;
10434    let deadline_ms =
10435        unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10436    let busy_gauges = match scope {
10437        DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10438        DrainScope::Endpoint(endpoint) => {
10439            declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10440        }
10441    };
10442
10443    // Admission gate first: route.open/commit and route REQUEST admission are closed
10444    // before the first quiescence check, so the outstanding count can only fall.
10445    let gate_started = Instant::now();
10446    let drain_target = match scope {
10447        DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10448        DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10449    }
10450    .map_err(SuperviseError::Forwarding)?;
10451    // The instant admission closed, and how long taking the forwarding write
10452    // lock to close it took. The timeout line reports only the quiescence
10453    // wait, so without this a drain that started late looked like one that
10454    // started on time.
10455    info!(
10456        module_id = %spec.module_id,
10457        ?reason,
10458        gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10459        connected = drain_target.is_some(),
10460        "module drain began; route admission closed"
10461    );
10462    if scope == DrainScope::Active {
10463        update_snapshot(snapshot, Some(&spec.module_id), |state| {
10464            state.state = ModuleState::Draining;
10465            state.draining_to_replace =
10466                matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10467            if let Some(enabled) = enabled {
10468                state.enabled = enabled;
10469            }
10470        })?;
10471    }
10472
10473    let Some(target) = drain_target.as_ref() else {
10474        // Nothing was sent: the module has no registered connection to carry
10475        // `module.draining` or a GOODBYE. The caller must not assume the child
10476        // was asked to stop.
10477        return Ok(StopNotice::NoConnection);
10478    };
10479    {
10480        send_module_draining(&spec.module_id, reason, deadline_ms, target);
10481        #[cfg(feature = "test-support")]
10482        forwarding.wait_before_route_closing_for_test().await;
10483        let routes = forwarding
10484            .endpoint_routes(target.endpoint)
10485            .map_err(SuperviseError::Forwarding)?;
10486        let routes_notified = routes.len();
10487        crate::control::send_route_control_pushes(
10488            forwarding,
10489            routes.clone(),
10490            ClientControlPush::RouteClosing {
10491                module_id: spec.module_id.clone(),
10492                channels: Vec::new(),
10493                reason,
10494            },
10495        );
10496        send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10497
10498        // `route.closing` was just sent above: from here on every return path,
10499        // including an early one, MUST send `route.closed` before propagating
10500        // anything else. A client holds `closing` as a promise that a verdict is
10501        // coming; leaving early without `closed` strands it waiting forever, since
10502        // `closing` carries no timeout of its own.
10503        let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10504        let wait_result = wait_for_forwarding_quiescence(
10505            forwarding,
10506            &spec.module_id,
10507            runtime,
10508            target.endpoint,
10509            drain_deadline,
10510            &busy_gauges,
10511            scope,
10512            &mut gauge_trace,
10513        )
10514        .await;
10515        let drained = drained_after_quiescence_wait(&wait_result);
10516        if let Err(err) = &wait_result {
10517            error!(
10518                module_id = %spec.module_id,
10519                ?reason,
10520                error = %err,
10521                "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10522            );
10523        } else if !drained {
10524            // Read before teardown releases the routes.
10525            let holdouts = forwarding
10526                .endpoint_drain_holdouts(target.endpoint)
10527                .unwrap_or_default();
10528            warn_drain_timed_out(
10529                &spec.module_id,
10530                drain_timeout,
10531                reason,
10532                &holdouts,
10533                &gauge_trace,
10534            );
10535        }
10536        crate::control::send_route_control_pushes(
10537            forwarding,
10538            routes,
10539            ClientControlPush::RouteClosed {
10540                module_id: spec.module_id.clone(),
10541                channels: Vec::new(),
10542                reason,
10543                drained,
10544                abandoned: target.abandoned_bindings.len() as u32,
10545                excluded_subscriptions: target.excluded_subscriptions,
10546                terminal: Some(terminal),
10547            },
10548        );
10549        wait_result?;
10550
10551        // `route.closed` has now been sent unconditionally above. From here the
10552        // remaining steps are cleanup (route + module GOODBYE) rather than a
10553        // promise the client is waiting on, but a lock-poisoned
10554        // `release_module_endpoint_routes` would otherwise skip the module
10555        // GOODBYE silently too -- send it before propagating the error.
10556        let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10557            Ok(routes) => routes,
10558            Err(err) => {
10559                warn!(
10560                    module_id = %spec.module_id,
10561                    ?reason,
10562                    error = %err,
10563                    "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10564                );
10565                send_module_goodbye(&spec.module_id, forwarding, target);
10566                return Err(SuperviseError::Forwarding(err));
10567            }
10568        };
10569        let route_goodbye_count = released_routes.len();
10570        send_route_goodbyes(forwarding, released_routes);
10571        send_module_goodbye(&spec.module_id, forwarding, target);
10572
10573        // The drain's happy path was previously silent: every emission above is
10574        // best-effort with only its failure arm logged, so "were consumers told"
10575        // was unprovable from the daemon log (surfaced by a 30-minute consumer
10576        // hang where the open question was exactly whether teardown notice went
10577        // out). One summary line makes that class decidable in one grep.
10578        info!(
10579            module_id = %spec.module_id,
10580            ?reason,
10581            routes_notified,
10582            route_goodbyes = route_goodbye_count,
10583            abandoned_reservations = target.abandoned_bindings.len(),
10584            excluded_subscriptions = target.excluded_subscriptions,
10585            drained,
10586            "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10587        );
10588    }
10589
10590    Ok(StopNotice::SentOverConnection)
10591}
10592
10593/// Wait for the freshly spawned child to take the ACTIVE slot for `module_id`,
10594/// the only slot a plain (non-swap) spawn can register into.
10595async fn wait_for_registration_after_reload(
10596    registry: &Registry,
10597    module_id: &str,
10598    snapshot: &SharedSnapshot,
10599    child: &mut SupervisedChild,
10600    wait: Duration,
10601) -> Result<RegistrationWaitOutcome, SuperviseError> {
10602    wait_for_slot_registration(
10603        registry,
10604        crate::registry::RegistrationSlot::Active(module_id),
10605        module_id,
10606        snapshot,
10607        child,
10608        wait,
10609    )
10610    .await
10611}
10612
10613/// Wait for `child` to register into `slot`, or to exit, or for `wait` to pass.
10614///
10615/// Keyed on the slot rather than the bare module id because during a swap the
10616/// id's active slot is already held by the incumbent: an id-keyed wait would
10617/// report the incumbent's registration as the candidate's and a candidate that
10618/// never registers would look registered. A swap candidate waits on
10619/// `crate::registry::RegistrationSlot::Candidate`.
10620async fn wait_for_slot_registration(
10621    registry: &Registry,
10622    slot: crate::registry::RegistrationSlot<'_>,
10623    module_id: &str,
10624    snapshot: &SharedSnapshot,
10625    child: &mut SupervisedChild,
10626    wait: Duration,
10627) -> Result<RegistrationWaitOutcome, SuperviseError> {
10628    let deadline = Instant::now() + wait;
10629    loop {
10630        if registry
10631            .registration(slot)
10632            .map_err(SuperviseError::Registry)?
10633            .is_some()
10634        {
10635            return Ok(RegistrationWaitOutcome::Registered);
10636        }
10637
10638        let now = Instant::now();
10639        if now >= deadline {
10640            return Ok(RegistrationWaitOutcome::TimedOut);
10641        }
10642        let remaining = deadline.saturating_duration_since(now);
10643        let poll = remaining.min(REGISTRY_RELEASE_POLL);
10644
10645        tokio::select! {
10646            wait_result = child.wait() => {
10647                let status = wait_result.map_err(|source| SuperviseError::Wait {
10648                    module_id: module_id.to_string(),
10649                    source,
10650                })?;
10651                return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10652                    snapshot,
10653                    child,
10654                    &status,
10655                )));
10656            }
10657            _ = sleep(poll) => {}
10658        }
10659    }
10660}
10661
10662fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10663    // A replacement process that exits before HELLO did not provide service, even
10664    // if it used status 0. Count it against the restart cap as a new-binary failure.
10665    if exit_report.kind != ExitKind::DeliberateSeverance {
10666        exit_report.kind = ExitKind::Crash;
10667    }
10668    exit_report
10669}
10670
10671async fn handle_reload_child_registration_failure(
10672    spec: &ModuleSpec,
10673    runtime: &SupervisorRuntimeConfig,
10674    registry: &Registry,
10675    process_liveness: &SupervisorProcessLiveness,
10676    snapshot: &SharedSnapshot,
10677    _child: &mut Option<SupervisedChild>,
10678    failure: ReloadRegistrationFailure,
10679) -> Result<(), SuperviseError> {
10680    let ReloadRegistrationFailure {
10681        exit_report,
10682        reason,
10683    } = failure;
10684    match on_child_exit(
10685        spec,
10686        runtime.restart_policy,
10687        registry,
10688        snapshot,
10689        &runtime.terminal_ring,
10690        &runtime.spawn_events,
10691        &runtime.child_roster,
10692        exit_report,
10693    )
10694    .await
10695    {
10696        NextAction::Stop {
10697            registration_released,
10698        } => {
10699            if registration_released {
10700                process_liveness.untrack_if_current(&spec.module_id, snapshot);
10701            }
10702        }
10703        NextAction::Restart { schedule } => {
10704            let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10705                schedule.delay
10706            });
10707            if let Some(schedule) = schedule {
10708                log_crash_respawn(&spec.module_id, schedule);
10709            }
10710            schedule_respawn(
10711                runtime,
10712                snapshot,
10713                &spec.module_id,
10714                delay,
10715                RespawnKind::Spawn,
10716            )?;
10717        }
10718    }
10719    Err(SuperviseError::ReloadFailed {
10720        module_id: spec.module_id.clone(),
10721        reason,
10722    })
10723}
10724
10725async fn handle_reload_spawn_failure(
10726    spec: &ModuleSpec,
10727    runtime: &SupervisorRuntimeConfig,
10728    process_liveness: &SupervisorProcessLiveness,
10729    snapshot: &SharedSnapshot,
10730    _child: &mut Option<SupervisedChild>,
10731    reason: String,
10732) -> Result<(), SuperviseError> {
10733    let now = Instant::now();
10734    let mut schedule = None;
10735    update_snapshot(snapshot, Some(&spec.module_id), |state| {
10736        clear_current_process_facts(state);
10737        if state.enabled {
10738            schedule = state.next_crash_restart(&runtime.restart_policy, now);
10739            state.state = if schedule.is_some() {
10740                ModuleState::Restarting
10741            } else {
10742                ModuleState::Failed
10743            };
10744        } else {
10745            state.state = ModuleState::Disabled;
10746        }
10747    })?;
10748    if let Some(schedule) = schedule {
10749        schedule_respawn(
10750            runtime,
10751            snapshot,
10752            &spec.module_id,
10753            schedule.delay,
10754            RespawnKind::Spawn,
10755        )?;
10756    } else {
10757        process_liveness.untrack_if_current(&spec.module_id, snapshot);
10758    }
10759    Err(SuperviseError::ReloadFailed {
10760        module_id: spec.module_id.clone(),
10761        reason,
10762    })
10763}
10764
10765fn control_flags() -> Flags {
10766    Flags::new(false, Priority::Passive, false)
10767}
10768
10769#[allow(clippy::too_many_arguments)]
10770async fn drain_optional_child(
10771    module_id: &str,
10772    protocol: ModuleProtocol,
10773    stop_notice: StopNotice,
10774    registry: &Registry,
10775    forwarding: Option<&ForwardingTable>,
10776    snapshot: &SharedSnapshot,
10777    terminal_ring: &Arc<Mutex<TerminalRing>>,
10778    spawn_events: &SpawnEventFeed,
10779    child: &mut Option<SupervisedChild>,
10780    drain_timeout: Duration,
10781    final_state: ModuleState,
10782    enabled: Option<bool>,
10783) -> Result<(), SuperviseError> {
10784    if let Some(child) = child.take() {
10785        drain_child_to_state(
10786            module_id,
10787            protocol,
10788            stop_notice,
10789            registry,
10790            forwarding,
10791            snapshot,
10792            terminal_ring,
10793            spawn_events,
10794            child,
10795            drain_timeout,
10796            final_state,
10797            enabled,
10798        )
10799        .await
10800    } else {
10801        update_snapshot(snapshot, Some(module_id), |state| {
10802            state.state = final_state;
10803            if let Some(enabled) = enabled {
10804                state.enabled = enabled;
10805            }
10806            clear_current_process_facts(state);
10807        })?;
10808        release_dead_registration(registry, forwarding, snapshot, module_id).await
10809    }
10810}
10811
10812#[allow(clippy::too_many_arguments)]
10813async fn drain_child_to_state(
10814    module_id: &str,
10815    _protocol: ModuleProtocol,
10816    stop_notice: StopNotice,
10817    registry: &Registry,
10818    forwarding: Option<&ForwardingTable>,
10819    snapshot: &SharedSnapshot,
10820    terminal_ring: &Arc<Mutex<TerminalRing>>,
10821    spawn_events: &SpawnEventFeed,
10822    mut child: SupervisedChild,
10823    drain_timeout: Duration,
10824    final_state: ModuleState,
10825    enabled: Option<bool>,
10826) -> Result<(), SuperviseError> {
10827    let protocol = child.protocol;
10828    update_snapshot(snapshot, Some(module_id), |state| {
10829        state.state = ModuleState::Draining;
10830        state.draining_to_replace = final_state == ModuleState::Restarting;
10831        if let Some(enabled) = enabled {
10832            state.enabled = enabled;
10833        }
10834    })?;
10835
10836    // The wait below is the same budget in every case; what differs is
10837    // whether anything has ASKED the child to stop before it starts. Only a
10838    // forwarding drain that reached the module's registered connection has
10839    // (`module.draining`, then a module GOODBYE). Every other child was told
10840    // nothing: a `protocol: "none"` module, which never registers; a subc
10841    // module spawned moments ago that has not sent HELLO yet; or a stop that
10842    // runs no forwarding drain. Without a signal the budget is only a delay
10843    // in front of SIGKILL -- and the not-yet-registered child is the worst
10844    // case, because it registers into a module that is already draining,
10845    // is never told, and is killed while healthy.
10846    if stop_notice != StopNotice::SentOverConnection {
10847        if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10848            info!(
10849                module_id,
10850                pid = child.pid,
10851                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10852                "module has no connection yet; requesting stop by signal"
10853            );
10854        }
10855        request_graceful_stop(module_id, &child);
10856    }
10857
10858    let exit_report = match timeout(drain_timeout, child.wait()).await {
10859        Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10860        Ok(Err(source)) => {
10861            fail_after_wait_error(
10862                module_id,
10863                snapshot,
10864                terminal_ring,
10865                spawn_events,
10866                format!("failed to wait during drain: {source}"),
10867            );
10868            return Err(SuperviseError::Wait {
10869                module_id: module_id.to_string(),
10870                source,
10871            });
10872        }
10873        Err(_) => {
10874            // Mirror the sibling arm above: state is already `Draining`, and an
10875            // error propagated from here would strand it there -- a state
10876            // `set_enabled(true)` cannot heal (`revive_terminal` matches only
10877            // `Failed | Stopped`), leaving an operator Restart as the only exit.
10878            // `Failed` before `?` keeps the module operator-visible and
10879            // revivable. Trigger is an ESRCH race (process exits between the
10880            // drain timeout firing and the kill) or a post-kill wait failure
10881            // (issue #34).
10882            //
10883            // Logged because the kill is otherwise visible only as signal 9 in
10884            // the terminal ring, and the budget it follows can be long enough
10885            // that consumers see a stretch of refusals with no stated cause.
10886            warn!(
10887                module_id,
10888                pid = child.pid,
10889                budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10890                reason = ?final_state,
10891                ?stop_notice,
10892                "drain budget expired before the module exited; killing it"
10893            );
10894            child.start_kill().map_err(|source| {
10895                fail_after_wait_error(
10896                    module_id,
10897                    snapshot,
10898                    terminal_ring,
10899                    spawn_events,
10900                    format!("failed to kill during drain: {source}"),
10901                );
10902                SuperviseError::Kill {
10903                    module_id: module_id.to_string(),
10904                    source,
10905                }
10906            })?;
10907            let status = child.wait().await.map_err(|source| {
10908                fail_after_wait_error(
10909                    module_id,
10910                    snapshot,
10911                    terminal_ring,
10912                    spawn_events,
10913                    format!("failed to wait after drain kill: {source}"),
10914                );
10915                SuperviseError::Wait {
10916                    module_id: module_id.to_string(),
10917                    source,
10918                }
10919            })?;
10920            classify_reaped_child_exit(snapshot, &child, &status)
10921        }
10922    };
10923
10924    let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10925    publish_after_terminal_record(
10926        module_id,
10927        || {
10928            record_terminal_with_detail(
10929                module_id,
10930                terminal_ring,
10931                spawn_events,
10932                &exit_report,
10933                terminal_disposition(final_state),
10934                detail,
10935            )
10936        },
10937        || {
10938            update_snapshot(snapshot, Some(module_id), |state| {
10939                state.state = final_state;
10940                if let Some(enabled) = enabled {
10941                    state.enabled = enabled;
10942                }
10943                clear_current_process_facts(state);
10944                state.last_exit = Some(exit_report.clone());
10945                if exit_report.kind == ExitKind::DeliberateSeverance {
10946                    state.lifetime_restarts += 1;
10947                }
10948            })
10949        },
10950    )?;
10951    child.drain_stderr(module_id).await;
10952
10953    release_dead_registration(registry, forwarding, snapshot, module_id).await
10954}
10955
10956/// Ask a child that nothing else has asked to stop, by signal.
10957///
10958/// A registered subc module is asked over its own connection: the drain sends
10959/// `route.closing`/`route.closed` to its consumers, a GOODBYE per route, then a
10960/// module GOODBYE, and the module stops itself. A module that speaks no subc
10961/// wire receives none of that, and neither does a subc module that has not
10962/// registered yet, so for them the drain budget would be pure delay in front of
10963/// a SIGKILL -- and for a process with a store to flush (JetStream is the
10964/// reason `protocol: "none"` exists) a SIGKILL turns every ordinary teardown
10965/// into a recovery on the next start.
10966///
10967/// NEVER CALLED FOR A MODULE THAT WAS TOLD OVER ITS CONNECTION, and that is a
10968/// rule rather than an optimisation: that module's graceful stop is already
10969/// running by the time its child is drained, and a signal would race it.
10970///
10971/// Best-effort by construction. A child that has already exited is the ordinary
10972/// case rather than an error (the kill lands on a reaped or exiting pid), so a
10973/// failure is logged at debug and the wait-then-kill below still decides the
10974/// outcome.
10975#[cfg(unix)]
10976fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10977    let Some(pid) = child
10978        .id()
10979        .and_then(|pid| i32::try_from(pid).ok())
10980        .and_then(rustix::process::Pid::from_raw)
10981    else {
10982        debug!(
10983            module_id,
10984            "no pid to signal for teardown; falling through to the drain wait"
10985        );
10986        return;
10987    };
10988    match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10989        Ok(()) => debug!(
10990            module_id,
10991            "sent SIGTERM to a module nothing else asked to stop"
10992        ),
10993        Err(err) => debug!(
10994            module_id,
10995            error = %err,
10996            "SIGTERM to module failed; the drain wait and kill still apply"
10997        ),
10998    }
10999}
11000
11001/// Windows has no SIGTERM and no portable stand-in for one. The graceful stops
11002/// Windows does offer need cooperation this supervisor cannot assume: a console
11003/// control event requires sharing a console with the child, and `WM_CLOSE`
11004/// requires the child to pump a message loop. A supervised server process does
11005/// neither, so there is nothing to send and teardown is the wait followed by the
11006/// kill. Emulating a signal here would mean inventing a stop protocol, which is
11007/// the thing `protocol: "none"` exists to avoid.
11008#[cfg(not(unix))]
11009fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11010    debug!(
11011        module_id,
11012        "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11013    );
11014}
11015
11016fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11017    match final_state {
11018        ModuleState::Stopped => TerminalDisposition::Stopped,
11019        ModuleState::Disabled => TerminalDisposition::Disabled,
11020        ModuleState::Restarting => TerminalDisposition::Restarting,
11021        ModuleState::Failed => TerminalDisposition::Failed,
11022        ModuleState::Starting
11023        | ModuleState::Running
11024        | ModuleState::Unresponsive
11025        | ModuleState::Draining => {
11026            unreachable!("terminal exits only finish in terminal or restarting states")
11027        }
11028    }
11029}
11030
11031/// Release a reaped child's registration before allowing another spawn.
11032///
11033/// EOF is not a process-lifetime signal: an inherited socket can stay open
11034/// indefinitely, and serial frame dispatch can be waiting on egress instead of
11035/// reading EOF. After the normal release grace, request connection close (which
11036/// cancels both reads and dispatch), then allow one more release grace for the
11037/// connection guard's forwarding cleanup. Never evict a different connection.
11038async fn release_dead_registration(
11039    registry: &Registry,
11040    forwarding: Option<&ForwardingTable>,
11041    snapshot: &SharedSnapshot,
11042    module_id: &str,
11043) -> Result<(), SuperviseError> {
11044    let result = async {
11045        let registration = registry
11046            .get_module(module_id)
11047            .map_err(SuperviseError::Registry)?;
11048        match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11049            Ok(()) => return Ok(()),
11050            Err(SuperviseError::RegistrationStillActive { .. }) => {}
11051            Err(err) => return Err(err),
11052        }
11053        let pid = lock_snapshot(snapshot)?.reaped_pid;
11054        if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11055            warn!(
11056                module_id,
11057                pid,
11058                connection_id = registration.connection_id.get(),
11059                "reaped module registration outlived release grace; closing dead connection"
11060            );
11061            forwarding.request_connection_close(
11062                registration.connection_id,
11063                CloseReason::new(
11064                    "supervised_process_reaped",
11065                    format!("module '{module_id}' pid {pid} exited"),
11066                ),
11067            );
11068            wait_for_slot_registration_release(
11069                registry,
11070                crate::registry::RegistrationSlot::Connection(registration.connection_id),
11071                REGISTRY_RELEASE_TIMEOUT,
11072            )
11073            .await?;
11074        }
11075        wait_for_registration_release(registry, module_id, Duration::ZERO).await
11076    }
11077    .await;
11078    if let Err(err) = &result {
11079        fail_snapshot(snapshot, Some(module_id), None);
11080        error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11081    }
11082    result
11083}
11084
11085/// Wait for the ACTIVE registration of `module_id` to go away, which is what a
11086/// plain stop or restart waits for before it spawns a replacement.
11087async fn wait_for_registration_release(
11088    registry: &Registry,
11089    module_id: &str,
11090    wait: Duration,
11091) -> Result<(), SuperviseError> {
11092    wait_for_slot_registration_release(
11093        registry,
11094        crate::registry::RegistrationSlot::Active(module_id),
11095        wait,
11096    )
11097    .await
11098}
11099
11100/// Wait for the registration in `slot` to go away.
11101///
11102/// Keyed on the slot rather than the bare module id because a successful swap
11103/// never empties the id's active slot (the promoted candidate is in it), so an
11104/// id-keyed wait for the incumbent's release would always time out. Draining a
11105/// swap's incumbent waits on `crate::registry::RegistrationSlot::Connection` with the
11106/// incumbent's connection instead.
11107async fn wait_for_slot_registration_release(
11108    registry: &Registry,
11109    slot: crate::registry::RegistrationSlot<'_>,
11110    wait: Duration,
11111) -> Result<(), SuperviseError> {
11112    let deadline = Instant::now() + wait;
11113    let mut release_events = registration_release_events().subscribe();
11114    let still_active = |registration: &crate::registry::ModuleRegistration| {
11115        SuperviseError::RegistrationStillActive {
11116            module_id: registration.manifest.module_id.clone(),
11117            waited: wait,
11118        }
11119    };
11120    loop {
11121        let _observed_generation = *release_events.borrow_and_update();
11122        let Some(registration) = registry
11123            .registration(slot)
11124            .map_err(SuperviseError::Registry)?
11125        else {
11126            return Ok(());
11127        };
11128
11129        let now = Instant::now();
11130        if now >= deadline {
11131            return Err(still_active(&registration));
11132        }
11133
11134        let remaining = deadline.saturating_duration_since(now);
11135        match timeout(remaining, release_events.changed()).await {
11136            Ok(Ok(())) | Ok(Err(_)) => {}
11137            Err(_) => return Err(still_active(&registration)),
11138        }
11139    }
11140}
11141
11142#[cfg(test)]
11143mod slot_registration_wait_tests {
11144    use super::*;
11145    use crate::registry::{ConnectionId, RegistrationSlot};
11146    use subc_protocol::manifest::ModuleManifest;
11147
11148    #[tokio::test]
11149    async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11150        let registry = Arc::new(Registry::default());
11151        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default());
11152        let runtime = supervisor.runtime_config();
11153        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11154        let spec = ModuleSpec {
11155            module_id: "enable-stale-registration".to_string(),
11156            program: PathBuf::from("/missing/enable-retry-test"),
11157            args: Vec::new(),
11158            env: Vec::new(),
11159            reserved: false,
11160            reserved_prefixes: Vec::new(),
11161            protocol: ModuleProtocol::Subc,
11162            overlap: Default::default(),
11163        };
11164        let connection = ConnectionId::new(90);
11165        registry
11166            .register_with_control_ops(
11167                ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11168                1,
11169                connection,
11170                Vec::new(),
11171            )
11172            .unwrap();
11173        let mut child = None;
11174        let err = set_child_enabled(
11175            &spec,
11176            &runtime,
11177            &registry,
11178            &supervisor.process_liveness,
11179            &snapshot,
11180            &mut child,
11181            true,
11182        )
11183        .await
11184        .unwrap_err();
11185        assert!(matches!(
11186            err,
11187            SuperviseError::RegistrationStillActive { .. }
11188        ));
11189        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11190        assert!(child.is_none());
11191        registry.deregister_connection(connection).unwrap();
11192        let err = set_child_enabled(
11193            &spec,
11194            &runtime,
11195            &registry,
11196            &supervisor.process_liveness,
11197            &snapshot,
11198            &mut child,
11199            true,
11200        )
11201        .await
11202        .unwrap_err();
11203        assert!(
11204            matches!(err, SuperviseError::Spawn { .. }),
11205            "second enable must attempt a spawn: {err}"
11206        );
11207        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11208    }
11209
11210    const INCUMBENT: u64 = 1;
11211    const CANDIDATE: u64 = 2;
11212
11213    fn swapped_registry() -> Arc<Registry> {
11214        let registry = Arc::new(Registry::default());
11215        let manifest = ModuleManifest::builder("m", "0.1.0").build();
11216        registry
11217            .register_with_control_ops(
11218                manifest.clone(),
11219                1,
11220                ConnectionId::new(INCUMBENT),
11221                Vec::new(),
11222            )
11223            .unwrap();
11224        registry
11225            .register_candidate_with_control_ops(
11226                manifest,
11227                1,
11228                ConnectionId::new(CANDIDATE),
11229                Vec::new(),
11230            )
11231            .unwrap();
11232        registry
11233    }
11234
11235    /// After a promotion the id's active slot is held by the new process, so an
11236    /// id-keyed wait for the incumbent's release can never succeed; the
11237    /// connection-keyed wait completes as soon as the incumbent deregisters.
11238    #[tokio::test]
11239    async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11240        let registry = swapped_registry();
11241        registry.promote_candidate("m").unwrap().unwrap();
11242
11243        assert!(matches!(
11244            wait_for_registration_release(&registry, "m", Duration::from_millis(50)).await,
11245            Err(SuperviseError::RegistrationStillActive { .. })
11246        ));
11247
11248        // Still held while the incumbent's connection has not deregistered.
11249        assert!(matches!(
11250            wait_for_slot_registration_release(
11251                &registry,
11252                RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11253                Duration::from_millis(50),
11254            )
11255            .await,
11256            Err(SuperviseError::RegistrationStillActive { .. })
11257        ));
11258
11259        let releaser = Arc::clone(&registry);
11260        let release = tokio::spawn(async move {
11261            sleep(Duration::from_millis(20)).await;
11262            releaser
11263                .deregister_connection(ConnectionId::new(INCUMBENT))
11264                .unwrap();
11265            notify_registration_release();
11266        });
11267        wait_for_slot_registration_release(
11268            &registry,
11269            RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11270            Duration::from_secs(5),
11271        )
11272        .await
11273        .expect("the incumbent's own registration is released");
11274        release.await.unwrap();
11275        assert!(registry.get_module("m").unwrap().is_some());
11276    }
11277
11278    /// The candidate slot is waited on separately from the active slot: the
11279    /// incumbent's registration neither holds up nor stands in for it.
11280    #[tokio::test]
11281    async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11282        let registry = swapped_registry();
11283        assert!(matches!(
11284            wait_for_slot_registration_release(
11285                &registry,
11286                RegistrationSlot::Candidate("m"),
11287                Duration::from_millis(50),
11288            )
11289            .await,
11290            Err(SuperviseError::RegistrationStillActive { .. })
11291        ));
11292        registry
11293            .deregister_connection(ConnectionId::new(CANDIDATE))
11294            .unwrap();
11295        wait_for_slot_registration_release(
11296            &registry,
11297            RegistrationSlot::Candidate("m"),
11298            Duration::from_millis(50),
11299        )
11300        .await
11301        .expect("a candidate slot with no candidate is released");
11302        assert!(registry
11303            .registration(RegistrationSlot::Active("m"))
11304            .unwrap()
11305            .is_some());
11306    }
11307}
11308
11309fn classify_exit(status: &ExitStatus) -> ExitReport {
11310    ExitReport {
11311        kind: if status.success() {
11312            ExitKind::Clean
11313        } else {
11314            ExitKind::Crash
11315        },
11316        code: status.code(),
11317        signal: exit_signal(status),
11318        at_ms: unix_ms_now(),
11319    }
11320}
11321
11322/// The terminal record for a module whose `wait()` call itself errored (e.g. the
11323/// child was already reaped out-of-band). There is no `ExitStatus` to read a code
11324/// or signal from -- `None`/`None` is the honest shape, not a guess -- but the
11325/// disposition still must be `Failed` so the terminal ring is not silently missing
11326/// an entry, matching what `fail_snapshot` records for this same arm.
11327fn wait_error_exit_report() -> ExitReport {
11328    ExitReport {
11329        kind: ExitKind::Crash,
11330        code: None,
11331        signal: None,
11332        at_ms: unix_ms_now(),
11333    }
11334}
11335
11336#[cfg(unix)]
11337fn exit_signal(status: &ExitStatus) -> Option<i32> {
11338    use std::os::unix::process::ExitStatusExt;
11339
11340    status.signal()
11341}
11342
11343#[cfg(not(unix))]
11344fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11345    None
11346}
11347
11348/// Give an operator-touched module its full crash budget back.
11349///
11350/// Named for the counter it used to zero; it now empties the in-window ring,
11351/// which is the same act. `lifetime_restarts` is untouched on purpose -- the
11352/// ledger of what happened survives every operator action.
11353fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11354    update_snapshot(snapshot, Some(module_id), |state| {
11355        state.clear_crash_restarts();
11356    })
11357}
11358
11359fn set_running(
11360    snapshot: &SharedSnapshot,
11361    child: &SupervisedChild,
11362    module_id: &str,
11363    spawn_events: &SpawnEventFeed,
11364) -> Result<(), SuperviseError> {
11365    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11366        module_id: Some(module_id.to_string()),
11367    })?;
11368    state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11369    if std::mem::take(&mut state.coalesced_restart_pending) {
11370        let generation = state.spawn_generation;
11371        info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11372    }
11373    state.drain_disposition_detail = None;
11374    state.spawn_failure = None;
11375    state.waiting_for_previous_pid = None;
11376    state.waiting_for_previous_detail = None;
11377    // Every caller of this is a plain spawn, which always uses the primary key;
11378    // a promoted swap candidate sets the flag itself after this returns.
11379    state.in_alternate_slot = false;
11380    state.configuration_updated_since_spawn = false;
11381    state.spawned_protocol = Some(child.protocol);
11382    state.state = ModuleState::Running;
11383    state.enabled = true;
11384    state.process_alive = true;
11385    state.pid = child.id();
11386    // Probe samples belong to one process. Keep module-level escalation and
11387    // late-answer history, but never charge a new child for its predecessor's misses.
11388    state.health.status = SupervisorHealthStatus::Unknown;
11389    state.health.consecutive_failures = 0;
11390    state.health.detail = None;
11391    state.health.metrics = None;
11392    state.health.last_probe_ms = None;
11393    #[cfg(windows)]
11394    {
11395        state.pipe_consumed = child
11396            .nonce_delivery
11397            .as_ref()
11398            .map(|delivery| delivery.consumed());
11399    }
11400    #[cfg(target_os = "macos")]
11401    {
11402        state.report_ready = Some(Arc::clone(&child.report_ready));
11403    }
11404    state.spawned_at_ms = Some(child.spawned_at_ms);
11405    state.exec_outcome = Some(Arc::clone(&child.exec.0));
11406    state.spawned_from = Some(child.spawned_from.clone());
11407    state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11408    state.spawned_file_identity = child.spawned_file_identity.clone();
11409    state.process_start_time = child.process_start_time;
11410    Ok(())
11411}
11412
11413fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11414    state.waiting_for_previous_pid = None;
11415    state.waiting_for_previous_detail = None;
11416    state.process_alive = false;
11417    state.spawned_protocol = None;
11418    state.pid = None;
11419    #[cfg(windows)]
11420    {
11421        state.pipe_consumed = None;
11422    }
11423    #[cfg(target_os = "macos")]
11424    {
11425        state.report_ready = None;
11426    }
11427    state.spawned_at_ms = None;
11428    state.exec_outcome = None;
11429    state.spawned_from = None;
11430    state.launch_spec_digest = None;
11431    state.spawned_file_identity = None;
11432    state.process_start_time = None;
11433    state.deliberate_severance = None;
11434}
11435
11436#[cfg(test)]
11437fn record_deliberate_severance(
11438    snapshot: &SharedSnapshot,
11439    identity: ProcessIdentity,
11440) -> Result<(), SuperviseError> {
11441    update_snapshot(snapshot, None, |state| {
11442        state.deliberate_severance = Some(identity);
11443    })
11444}
11445
11446fn apply_deliberate_severance_marker(
11447    snapshot: &SharedSnapshot,
11448    exited_identity: Option<ProcessIdentity>,
11449    mut exit_report: ExitReport,
11450) -> ExitReport {
11451    let marker = lock_snapshot(snapshot)
11452        .ok()
11453        .and_then(|mut state| state.deliberate_severance.take());
11454    if marker.is_some() && marker == exited_identity {
11455        exit_report.kind = ExitKind::DeliberateSeverance;
11456    }
11457    exit_report
11458}
11459
11460fn classify_reaped_child_exit(
11461    snapshot: &SharedSnapshot,
11462    child: &SupervisedChild,
11463    status: &ExitStatus,
11464) -> ExitReport {
11465    let _ = update_snapshot(snapshot, None, |state| {
11466        state.reaped_pid = Some(child.pid);
11467        state.spawn_failure = child.spawn_failure.clone();
11468    });
11469    apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11470}
11471
11472fn fail_snapshot(
11473    snapshot: &SharedSnapshot,
11474    module_id: Option<&str>,
11475    last_exit: Option<ExitReport>,
11476) {
11477    if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11478        state.state = ModuleState::Failed;
11479        clear_current_process_facts(state);
11480        if let Some(last_exit) = last_exit {
11481            state.last_exit = Some(last_exit);
11482        }
11483    }) {
11484        error!(error = %err, "failed to mark supervisor state failed");
11485    }
11486}
11487
11488fn update_snapshot(
11489    snapshot: &SharedSnapshot,
11490    module_id: Option<&str>,
11491    update: impl FnOnce(&mut SupervisorSnapshot),
11492) -> Result<(), SuperviseError> {
11493    let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11494        module_id: module_id.map(ToOwned::to_owned),
11495    })?;
11496    update(&mut state);
11497    Ok(())
11498}
11499
11500const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11501
11502fn lock_snapshot_for_control<'a>(
11503    snapshot: &'a SharedSnapshot,
11504    module_id: &str,
11505    caller: &'static str,
11506) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11507    let started_at = Instant::now();
11508    let guard = lock_snapshot(snapshot)?;
11509    let waited = started_at.elapsed();
11510    if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11511        warn!(
11512            module_id = %module_id,
11513            waited_ms = waited.as_millis() as u64,
11514            caller = %caller,
11515            "slow snapshot lock"
11516        );
11517    }
11518    Ok(guard)
11519}
11520
11521fn lock_snapshot(
11522    snapshot: &SharedSnapshot,
11523) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11524    snapshot
11525        .lock()
11526        .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11527}
11528
11529#[cfg(test)]
11530mod terminal_history_tests {
11531    use std::{
11532        path::PathBuf,
11533        sync::Arc,
11534        time::{Duration, Instant},
11535    };
11536
11537    use tokio::{sync::mpsc, time::sleep};
11538
11539    use super::{
11540        apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11541        drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11542        lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11543        reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11544        ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11545        RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11546        SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11547        SupervisorHealthStatus, SupervisorSnapshot,
11548    };
11549    // The supervisor's clock, distinct from the `std::time::Instant` these tests
11550    // use for their own wall-clock deadlines: crash-restart instants must be on
11551    // the same clock the production code stamps them with, which is tokio's (and
11552    // is what `start_paused` tests can move).
11553    use super::Instant as ClockInstant;
11554    use crate::{
11555        registry::Registry,
11556        terminal_ring::{TerminalRing, TerminalRingConfig},
11557    };
11558    use std::sync::Mutex;
11559    use subc_control::TerminalDisposition;
11560
11561    /// See the twin in `control.rs` for why this derives the path from
11562    /// `current_exe()` and why the existence check is here: `--lib` alone does
11563    /// not build `[[bin]]` targets, and a bare spawn then fails with a raw
11564    /// `NotFound` that reads as a broken test rather than an unbuilt dependency.
11565    pub(super) fn fake_aft_stub_path() -> PathBuf {
11566        let mut path = std::env::current_exe().expect("current_exe available in tests");
11567        path.pop();
11568        path.pop();
11569        path.push(if cfg!(windows) {
11570            "fake-aft-stub.exe"
11571        } else {
11572            "fake-aft-stub"
11573        });
11574        assert!(
11575            path.exists(),
11576            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11577             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11578            path.display()
11579        );
11580        path
11581    }
11582
11583    #[test]
11584    fn reserved_never_spawned_refuses_every_hello() {
11585        // The canary hole: a reserved id whose module has never spawned had NO
11586        // gate entry and admitted anyone -- the reservation protected the nonce
11587        // holder, not the NAME. Now the entry is present with no legitimate
11588        // holder and refuses all comers.
11589        let supervisor = SupervisorHandle::default();
11590        supervisor.apply_identity_configuration(&ModuleSpec {
11591            module_id: "never-spawned".to_string(),
11592            program: PathBuf::from("/usr/bin/false"),
11593            args: Vec::new(),
11594            env: Vec::new(),
11595            reserved: true,
11596            reserved_prefixes: Vec::new(),
11597            protocol: ModuleProtocol::Subc,
11598            overlap: Default::default(),
11599        });
11600        assert!(
11601            supervisor
11602                .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11603                .is_err(),
11604            "forged nonce must refuse on a reserved never-spawned id"
11605        );
11606        assert!(
11607            supervisor
11608                .reserved_hello_verification("never-spawned", None)
11609                .is_err(),
11610            "absent nonce must refuse on a reserved never-spawned id"
11611        );
11612        // And a real spawn nonce minted later admits exactly that nonce.
11613        supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11614        supervisor.apply_identity_configuration(&ModuleSpec {
11615            module_id: "never-spawned".to_string(),
11616            program: PathBuf::from("/usr/bin/false"),
11617            args: Vec::new(),
11618            env: Vec::new(),
11619            reserved: true,
11620            reserved_prefixes: Vec::new(),
11621            protocol: ModuleProtocol::Subc,
11622            overlap: Default::default(),
11623        });
11624        assert!(supervisor
11625            .reserved_hello_verification("never-spawned", Some("minted"))
11626            .is_ok());
11627        assert!(supervisor
11628            .reserved_hello_verification("never-spawned", Some("forged"))
11629            .is_err());
11630    }
11631
11632    /// Put `count` crash restarts on a snapshot's ring as if they had all just
11633    /// happened, which is what "spent budget" looks like to every reader.
11634    fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11635        let now = ClockInstant::now();
11636        for _ in 0..count {
11637            state.crash_restarts.push_back(now);
11638        }
11639    }
11640
11641    /// Age the oldest recorded restart out of `window`, standing in for the hours
11642    /// that would otherwise have to pass. Injecting the instant is the point: a
11643    /// test that slept a real window would take ten minutes and still prove less.
11644    fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11645        let aged = state
11646            .crash_restarts
11647            .front()
11648            .expect("a crash restart must be recorded before it can be aged")
11649            .checked_sub(window + Duration::from_secs(1))
11650            .expect("the test clock is far enough from its origin to age an instant");
11651        state.crash_restarts[0] = aged;
11652    }
11653
11654    fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11655        let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11656        seed_crash_restarts(&mut state, count);
11657        state
11658    }
11659
11660    #[test]
11661    fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11662        let policy = RestartPolicy::new(3, Duration::ZERO);
11663        let now = ClockInstant::now();
11664        assert!(daemon_will_restart(
11665            &mut snapshot_with_restarts(true, 2),
11666            &policy,
11667            now
11668        ));
11669        assert!(!daemon_will_restart(
11670            &mut snapshot_with_restarts(true, 3),
11671            &policy,
11672            now
11673        ));
11674        assert!(!daemon_will_restart(
11675            &mut snapshot_with_restarts(false, 0),
11676            &policy,
11677            now
11678        ));
11679    }
11680
11681    #[test]
11682    fn crash_restart_backoff_escalates_with_in_window_count() {
11683        let policy = RestartPolicy::new(4, Duration::from_millis(100))
11684            .with_max_backoff(Duration::from_secs(30));
11685        let now = ClockInstant::now();
11686        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11687        let schedules = (0..4)
11688            .map(|_| {
11689                state
11690                    .next_crash_restart(&policy, now)
11691                    .expect("the test policy allows four crash restarts")
11692            })
11693            .collect::<Vec<_>>();
11694
11695        assert_eq!(
11696            schedules
11697                .iter()
11698                .map(|schedule| schedule.restart_in_window)
11699                .collect::<Vec<_>>(),
11700            vec![0, 1, 2, 3]
11701        );
11702        assert_eq!(
11703            schedules
11704                .iter()
11705                .map(|schedule| schedule.delay)
11706                .collect::<Vec<_>>(),
11707            vec![
11708                Duration::from_millis(100),
11709                Duration::from_secs(1),
11710                Duration::from_secs(10),
11711                Duration::from_secs(30),
11712            ]
11713        );
11714    }
11715
11716    #[test]
11717    fn crash_restart_backoff_resets_after_ring_clear() {
11718        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11719        let now = ClockInstant::now();
11720        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11721        assert_eq!(
11722            state.next_crash_restart(&policy, now).unwrap().delay,
11723            Duration::from_millis(100)
11724        );
11725        assert_eq!(
11726            state.next_crash_restart(&policy, now).unwrap().delay,
11727            Duration::from_secs(1)
11728        );
11729
11730        state.clear_crash_restarts();
11731        let schedule = state
11732            .next_crash_restart(&policy, now)
11733            .expect("a cleared ring must allow another restart");
11734        assert_eq!(schedule.restart_in_window, 0);
11735        assert_eq!(schedule.delay, Duration::from_millis(100));
11736    }
11737
11738    #[test]
11739    fn crash_restart_backoff_ignores_aged_restarts() {
11740        let policy = RestartPolicy::new(3, Duration::from_millis(100));
11741        let now = ClockInstant::now();
11742        let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11743        state
11744            .next_crash_restart(&policy, now)
11745            .expect("the first restart is allowed");
11746        state
11747            .next_crash_restart(&policy, now)
11748            .expect("the second restart is allowed");
11749        state.crash_restarts[0] = now
11750            .checked_sub(policy.window + Duration::from_secs(1))
11751            .expect("the fake clock can age a restart past the window");
11752
11753        let schedule = state
11754            .next_crash_restart(&policy, now)
11755            .expect("an aged restart must release its slot");
11756        assert_eq!(schedule.restart_in_window, 1);
11757        assert_eq!(schedule.delay, Duration::from_secs(1));
11758        assert_eq!(state.crash_restarts.len(), 2);
11759    }
11760
11761    /// The budget is a rate: the same three spent restarts refuse a respawn
11762    /// while they are recent and allow one once they have aged past the window.
11763    /// Nothing about the module changed in between, which is the whole point.
11764    #[test]
11765    fn a_budget_spent_before_the_window_no_longer_refuses() {
11766        let policy = RestartPolicy::new(3, Duration::ZERO);
11767        let mut state = snapshot_with_restarts(true, 3);
11768        let now = ClockInstant::now();
11769        assert!(!daemon_will_restart(&mut state, &policy, now));
11770
11771        assert!(daemon_will_restart(
11772            &mut state,
11773            &policy,
11774            now + policy.window + Duration::from_secs(1)
11775        ));
11776        assert!(
11777            state.crash_restarts.is_empty(),
11778            "reading the budget must drop the instants that left the window"
11779        );
11780    }
11781
11782    fn module_with_recovery_snapshot(
11783        state: ModuleState,
11784        enabled: bool,
11785        restart_count: u32,
11786    ) -> SupervisedModule {
11787        let registry = Arc::new(Registry::default());
11788        let supervisor =
11789            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(3, Duration::ZERO));
11790        let runtime = supervisor.runtime_config();
11791        let spec = ModuleSpec {
11792            module_id: "recovery-snapshot".to_string(),
11793            program: fake_aft_stub_path(),
11794            args: Vec::new(),
11795            env: Vec::new(),
11796            reserved: false,
11797            reserved_prefixes: Vec::new(),
11798            protocol: ModuleProtocol::Subc,
11799            overlap: Default::default(),
11800        };
11801        let mut snapshot = SupervisorSnapshot::new(state, enabled);
11802        seed_crash_restarts(&mut snapshot, restart_count);
11803        // These tests read synthetic snapshots. A real child and monitor would
11804        // race those reads by replacing the requested state during startup.
11805        let (commands, _rx) = mpsc::channel(4);
11806        SupervisedModule {
11807            inner: Arc::new(SupervisedModuleInner {
11808                module_id: spec.module_id.clone(),
11809                registry,
11810                snapshot: Arc::new(Mutex::new(snapshot)),
11811                configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11812                    spec,
11813                    health: runtime.health,
11814                })),
11815                stderr_ring: runtime.stderr_ring,
11816                terminal_ring: runtime.terminal_ring,
11817                commands,
11818                monitor: Mutex::new(None),
11819                restart_policy: runtime.restart_policy,
11820                effective_drain_timeout: runtime.effective_drain_timeout,
11821                provenance_probe: supervisor.provenance_probe.clone(),
11822            }),
11823        }
11824    }
11825
11826    #[cfg(target_os = "linux")]
11827    #[tokio::test]
11828    async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11829        let supervisor =
11830            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11831                .with_cgroup_placement(None);
11832        let result = supervisor.spawn(ModuleSpec {
11833            module_id: "no-cgroup-placement".to_string(),
11834            program: fake_aft_stub_path(),
11835            args: Vec::new(),
11836            env: Vec::new(),
11837            reserved: false,
11838            reserved_prefixes: Vec::new(),
11839            protocol: ModuleProtocol::Subc,
11840            overlap: Default::default(),
11841        });
11842
11843        assert!(
11844            result.is_ok(),
11845            "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11846        );
11847    }
11848
11849    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11850    async fn undecided_snapshot_uses_shared_restart_predicate() {
11851        assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11852            .will_recover_after_connection_loss()
11853            .unwrap());
11854        assert!(
11855            !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11856                .will_recover_after_connection_loss()
11857                .unwrap()
11858        );
11859    }
11860
11861    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11862    async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11863        assert!(
11864            module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11865                .will_recover_after_connection_loss()
11866                .unwrap()
11867        );
11868    }
11869
11870    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11871    async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11872        assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11873            .will_recover_after_connection_loss()
11874            .unwrap());
11875        assert!(
11876            !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11877                .will_recover_after_connection_loss()
11878                .unwrap()
11879        );
11880    }
11881
11882    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11883    async fn warming_snapshot_is_limited_to_startup_phases() {
11884        for state in [
11885            ModuleState::Starting,
11886            ModuleState::Running,
11887            ModuleState::Restarting,
11888        ] {
11889            assert!(
11890                module_with_recovery_snapshot(state, true, 0)
11891                    .is_warming()
11892                    .unwrap(),
11893                "{state:?} should be warming"
11894            );
11895        }
11896        for state in [
11897            ModuleState::Unresponsive,
11898            ModuleState::Draining,
11899            ModuleState::Stopped,
11900            ModuleState::Failed,
11901            ModuleState::Disabled,
11902        ] {
11903            assert!(
11904                !module_with_recovery_snapshot(state, true, 0)
11905                    .is_warming()
11906                    .unwrap(),
11907                "{state:?} should not be warming"
11908            );
11909        }
11910    }
11911
11912    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11913    async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11914        let registry = Arc::new(Registry::default());
11915        let supervisor =
11916            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO));
11917        let module = supervisor
11918            .spawn(ModuleSpec {
11919                module_id: "terminal-history".to_string(),
11920                program: fake_aft_stub_path(),
11921                args: Vec::new(),
11922                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11923                reserved: false,
11924                reserved_prefixes: Vec::new(),
11925                protocol: ModuleProtocol::Subc,
11926                overlap: Default::default(),
11927            })
11928            .unwrap();
11929
11930        let deadline = Instant::now() + Duration::from_secs(5);
11931        loop {
11932            let history = module.terminal_history();
11933            if history.entries.len() == 2 {
11934                assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11935                assert_eq!(history.dropped, 0);
11936                assert_eq!(
11937                    history
11938                        .entries
11939                        .iter()
11940                        .map(|entry| entry.exit_code)
11941                        .collect::<Vec<_>>(),
11942                    vec![Some(23), Some(23)]
11943                );
11944                assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11945                return;
11946            }
11947            assert!(
11948                Instant::now() < deadline,
11949                "module did not retain two terminal exits: {history:?}"
11950            );
11951            sleep(Duration::from_millis(10)).await;
11952        }
11953    }
11954
11955    /// A disable issued while a crash respawn is still backing off must preempt
11956    /// that respawn: the operator's stop wins, the disable must not queue behind
11957    /// the backoff, and the module must never come back up afterwards.
11958    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11959    async fn disable_during_crash_backoff_cancels_pending_respawn() {
11960        let backoff = Duration::from_secs(2);
11961        let supervisor = Supervisor::new_for_test(
11962            Arc::new(Registry::default()),
11963            RestartPolicy::new(10, backoff),
11964        );
11965        let module = supervisor
11966            .spawn(ModuleSpec {
11967                module_id: "disable-during-backoff".to_string(),
11968                program: fake_aft_stub_path(),
11969                args: Vec::new(),
11970                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11971                reserved: false,
11972                reserved_prefixes: Vec::new(),
11973                protocol: ModuleProtocol::Subc,
11974                overlap: Default::default(),
11975            })
11976            .unwrap();
11977
11978        // Wait for the first crash to put the module into its backoff window.
11979        let deadline = Instant::now() + Duration::from_secs(5);
11980        loop {
11981            if module.status().unwrap().state == ModuleState::Restarting {
11982                break;
11983            }
11984            assert!(
11985                Instant::now() < deadline,
11986                "module never entered the crash backoff"
11987            );
11988            sleep(Duration::from_millis(10)).await;
11989        }
11990
11991        let started = Instant::now();
11992        module.set_enabled(false).await.unwrap();
11993        let waited = started.elapsed();
11994
11995        assert!(
11996            waited < backoff / 2,
11997            "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11998        );
11999        assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12000
12001        // Outlast the backoff: the respawn it was counting down to must never run.
12002        sleep(backoff + Duration::from_millis(500)).await;
12003        let status = module.status().unwrap();
12004        assert_eq!(status.state, ModuleState::Disabled);
12005        assert_eq!(
12006            status.spawn_generation, 1,
12007            "module respawned after the operator disabled it"
12008        );
12009    }
12010
12011    /// A `protocol: "none"` child that stays parked and exits 0 on SIGTERM:
12012    /// the shape of nats-server, the program this rule exists for.
12013    #[cfg(unix)]
12014    fn protocol_none_sigterm_exits_clean_spec(
12015        module_id: &str,
12016        dir: &std::path::Path,
12017    ) -> (ModuleSpec, PathBuf, PathBuf) {
12018        let ready = dir.join("ready");
12019        let marker = dir.join("sigterm");
12020        let spec = ModuleSpec {
12021            module_id: module_id.to_string(),
12022            program: fake_aft_stub_path(),
12023            args: Vec::new(),
12024            env: vec![
12025                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12026                (
12027                    "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12028                    marker.display().to_string(),
12029                ),
12030                (
12031                    "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12032                    ready.display().to_string(),
12033                ),
12034            ],
12035            reserved: false,
12036            reserved_prefixes: Vec::new(),
12037            protocol: ModuleProtocol::None,
12038            overlap: Default::default(),
12039        };
12040        (spec, ready, marker)
12041    }
12042
12043    /// Wait for a file the child writes, so a signal is never sent before the
12044    /// child's SIGTERM handler is installed (the default disposition would
12045    /// kill it by signal and the exit would not be clean).
12046    #[cfg(unix)]
12047    async fn wait_for_file(path: &std::path::Path) {
12048        let deadline = Instant::now() + Duration::from_secs(10);
12049        while !path.exists() {
12050            assert!(
12051                Instant::now() < deadline,
12052                "{} never appeared",
12053                path.display()
12054            );
12055            sleep(Duration::from_millis(10)).await;
12056        }
12057    }
12058
12059    /// A protocol-none module that exits 0 because something OUTSIDE the
12060    /// supervisor sent it SIGTERM is respawned, and the terminal record carries
12061    /// the crash-path disposition rather than `stopped`.
12062    #[cfg(unix)]
12063    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12064    async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12065        let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12066        let (spec, ready, marker) =
12067            protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12068        let supervisor = Supervisor::new_for_test(
12069            Arc::new(Registry::default()),
12070            RestartPolicy::new(3, Duration::ZERO),
12071        );
12072        let module = supervisor.spawn(spec).unwrap();
12073        wait_for_file(&ready).await;
12074        // The ready file proves the child installed its SIGTERM handler, not
12075        // that the supervisor has processed the privacy trampoline's exec
12076        // acknowledgement. On macOS status withholds the pid until then.
12077        let deadline = Instant::now() + Duration::from_secs(10);
12078        let first_pid = loop {
12079            let status = module.status().unwrap();
12080            if status.state == ModuleState::Running {
12081                if let Some(pid) = status.pid {
12082                    break pid;
12083                }
12084            }
12085            assert!(
12086                Instant::now() < deadline,
12087                "a running module must report its pid after exec confirmation: {status:?}"
12088            );
12089            sleep(Duration::from_millis(10)).await;
12090        };
12091
12092        rustix::process::kill_process(
12093            rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12094            rustix::process::Signal::TERM,
12095        )
12096        .unwrap();
12097
12098        let deadline = Instant::now() + Duration::from_secs(10);
12099        let respawned = loop {
12100            let status = module.status().unwrap();
12101            if status.state == ModuleState::Running
12102                && status.pid.is_some_and(|pid| pid != first_pid)
12103            {
12104                break status;
12105            }
12106            assert!(
12107                Instant::now() < deadline,
12108                "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12109            );
12110            sleep(Duration::from_millis(10)).await;
12111        };
12112        assert_eq!(respawned.spawn_generation, 2);
12113        assert!(
12114            marker.exists(),
12115            "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12116        );
12117
12118        let history = module.terminal_history();
12119        assert_eq!(history.entries.len(), 1, "{history:?}");
12120        let entry = &history.entries[0];
12121        assert_eq!(entry.exit_code, Some(0));
12122        assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12123        assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12124
12125        module.stop().await.unwrap();
12126    }
12127
12128    /// Repeated unrequested clean exits of a protocol-none module spend the
12129    /// restart budget exactly as crashes do, and the module ends `failed` with
12130    /// the budget named.
12131    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12132    async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12133        let supervisor = Supervisor::new_for_test(
12134            Arc::new(Registry::default()),
12135            RestartPolicy::new(1, Duration::ZERO),
12136        );
12137        let module = supervisor
12138            .spawn(ModuleSpec {
12139                module_id: "none-clean-exit-budget".to_string(),
12140                program: fake_aft_stub_path(),
12141                args: Vec::new(),
12142                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12143                reserved: false,
12144                reserved_prefixes: Vec::new(),
12145                protocol: ModuleProtocol::None,
12146                overlap: Default::default(),
12147            })
12148            .unwrap();
12149
12150        // Failed follows the terminal write; this deadline only bounds a hang,
12151        // not an assumed duration for the two launches or their exit recording.
12152        let deadline = Instant::now() + Duration::from_secs(10);
12153        loop {
12154            let status = module.status().unwrap();
12155            if status.state == ModuleState::Failed {
12156                break;
12157            }
12158            assert!(
12159                Instant::now() < deadline,
12160                "module never exhausted its budget: {status:?} {:?}",
12161                module.terminal_history()
12162            );
12163            sleep(Duration::from_millis(10)).await;
12164        }
12165        let history = module.terminal_history();
12166        assert_eq!(
12167            history
12168                .entries
12169                .iter()
12170                .map(|entry| (entry.exit_code, entry.disposition.clone()))
12171                .collect::<Vec<_>>(),
12172            vec![
12173                (Some(0), TerminalDisposition::Restarting),
12174                (Some(0), TerminalDisposition::Failed),
12175            ]
12176        );
12177        let detail = history.entries[1]
12178            .disposition_detail
12179            .as_deref()
12180            .expect("a budget failure names the budget");
12181        assert!(detail.contains("max_restarts=1"), "{detail}");
12182        assert_eq!(module.status().unwrap().spawn_generation, 2);
12183    }
12184
12185    #[test]
12186    fn restart_budget_failure_is_published_after_its_terminal_record() {
12187        use std::io::{Read, Write};
12188        let supervisor = Supervisor::new_for_test(
12189            Arc::new(Registry::default()),
12190            RestartPolicy::new(0, Duration::ZERO),
12191        );
12192        let runtime = supervisor.runtime_config();
12193        let spec = ModuleSpec {
12194            protocol: ModuleProtocol::None,
12195            ..windowed_crash_spec("budget-publication-order")
12196        };
12197        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12198            ModuleState::Running,
12199            true,
12200        )));
12201        lock_snapshot(&snapshot).unwrap().process_alive = true;
12202        let reaped_snapshot = snapshot.clone();
12203        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12204        let history = runtime.terminal_ring.clone();
12205        let reap = std::thread::spawn(move || {
12206            tokio::runtime::Builder::new_current_thread()
12207                .enable_all()
12208                .build()
12209                .unwrap()
12210                .block_on(on_child_exit(
12211                    &spec,
12212                    runtime.restart_policy,
12213                    &supervisor.registry,
12214                    &reaped_snapshot,
12215                    &runtime.terminal_ring,
12216                    &runtime.spawn_events,
12217                    &runtime.child_roster,
12218                    ExitReport {
12219                        kind: ExitKind::Clean,
12220                        code: Some(0),
12221                        signal: None,
12222                        at_ms: 1,
12223                    },
12224                ))
12225        });
12226        let mut reached = [0];
12227        barrier.read_exact(&mut reached).unwrap();
12228        assert_eq!(&reached, b"R");
12229        let (before_state, before_alive) = {
12230            let state = lock_snapshot(&snapshot).unwrap();
12231            (state.state, state.process_alive)
12232        };
12233        let before_history = history.lock().unwrap().snapshot();
12234        barrier.write_all(b"X").unwrap();
12235        assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12236        assert!(before_history.entries.is_empty());
12237        assert_ne!(
12238            before_state,
12239            ModuleState::Failed,
12240            "Failed was visible before its terminal record could be written"
12241        );
12242        assert!(before_alive);
12243        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12244        let history = history.lock().unwrap().snapshot();
12245        assert_eq!(history.entries.len(), 1);
12246        assert_eq!(history.entries[0].exit_code, Some(0));
12247        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12248    }
12249
12250    #[test]
12251    fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12252        use std::io::{Read, Write};
12253
12254        let supervisor = Supervisor::new_for_test(
12255            Arc::new(Registry::default()),
12256            RestartPolicy::new(3, Duration::ZERO),
12257        );
12258        let mut runtime = supervisor.runtime_config();
12259        let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12260        let snapshot = module.inner.snapshot.clone();
12261        lock_snapshot(&snapshot).unwrap().process_alive = true;
12262        runtime.terminal_ring = module.inner.terminal_ring.clone();
12263        let spec = windowed_crash_spec(module.module_id());
12264        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12265        let reaped_snapshot = snapshot.clone();
12266        let reap = std::thread::spawn(move || {
12267            tokio::runtime::Builder::new_current_thread()
12268                .enable_all()
12269                .build()
12270                .unwrap()
12271                .block_on(on_child_exit(
12272                    &spec,
12273                    runtime.restart_policy,
12274                    &supervisor.registry,
12275                    &reaped_snapshot,
12276                    &runtime.terminal_ring,
12277                    &runtime.spawn_events,
12278                    &runtime.child_roster,
12279                    ExitReport {
12280                        kind: ExitKind::Crash,
12281                        code: Some(2),
12282                        signal: None,
12283                        at_ms: 1,
12284                    },
12285                ))
12286        });
12287
12288        // Hold the crash handler between its budget decision and publication.
12289        // Connection cleanup must still see the recovery that this crash chose,
12290        // even when that recovery consumes the last available restart slot.
12291        let mut reached = [0];
12292        barrier.read_exact(&mut reached).unwrap();
12293        assert_eq!(&reached, b"R");
12294        let before_status = module.status().unwrap();
12295        let before_history = module.terminal_history();
12296        let before_recovers = module.will_recover_after_connection_loss().unwrap();
12297        barrier.write_all(b"X").unwrap();
12298        assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12299
12300        assert_eq!(before_status.state, ModuleState::Running);
12301        assert!(before_status.process_alive);
12302        assert_eq!(before_status.restart_count, 3);
12303        assert!(before_history.entries.is_empty());
12304        assert!(
12305            before_recovers,
12306            "connection cleanup would report a terminal close for a scheduled crash restart"
12307        );
12308        let restarting = module.status().unwrap();
12309        assert_eq!(restarting.state, ModuleState::Restarting);
12310        assert_eq!(restarting.restart_count, 3);
12311        assert!(module.will_recover_after_connection_loss().unwrap());
12312        let history = module.terminal_history();
12313        assert_eq!(history.entries.len(), 1);
12314        assert_eq!(
12315            history.entries[0].disposition,
12316            TerminalDisposition::Restarting
12317        );
12318
12319        // Once the replacement is running, its next crash must be judged
12320        // against the exhausted budget rather than the previous recovery.
12321        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12322            state.state = ModuleState::Running;
12323            state.respawn_pending = false;
12324        })
12325        .unwrap();
12326        assert!(!module.will_recover_after_connection_loss().unwrap());
12327        update_snapshot(&snapshot, Some(module.module_id()), |state| {
12328            state.enabled = false;
12329            state.respawn_pending = true;
12330        })
12331        .unwrap();
12332        assert!(!module.will_recover_after_connection_loss().unwrap());
12333    }
12334
12335    fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12336        use std::net::{TcpListener, TcpStream};
12337        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12338        let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12339        let (barrier, _) = listener.accept().unwrap();
12340        for stream in [&writer, &barrier] {
12341            stream
12342                .set_read_timeout(Some(Duration::from_secs(10)))
12343                .unwrap();
12344            stream
12345                .set_write_timeout(Some(Duration::from_secs(10)))
12346                .unwrap();
12347        }
12348        *events.1.lock().unwrap() = Some(writer);
12349        barrier
12350    }
12351
12352    pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12353        use std::io::{Read, Write};
12354        let barrier = events.1.lock().unwrap().take();
12355        if let Some(mut barrier) = barrier {
12356            barrier.write_all(b"R").unwrap();
12357            let mut release = [0];
12358            barrier.read_exact(&mut release).unwrap();
12359            assert_eq!(&release, b"X");
12360        }
12361    }
12362
12363    #[test]
12364    fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12365        use std::io::{Read, Write};
12366
12367        let supervisor = Supervisor::new_for_test(
12368            Arc::new(Registry::default()),
12369            RestartPolicy::new(0, Duration::ZERO),
12370        );
12371        let mut runtime = supervisor.runtime_config();
12372        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12373        let snapshot = module.inner.snapshot.clone();
12374        runtime.terminal_ring = module.inner.terminal_ring.clone();
12375        let spec = ModuleSpec {
12376            module_id: module.module_id().into(),
12377            program: fake_aft_stub_path(),
12378            protocol: ModuleProtocol::None,
12379            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12380            ..windowed_crash_spec(module.module_id())
12381        };
12382        let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12383        let reap = std::thread::spawn(move || {
12384            tokio::runtime::Builder::new_current_thread()
12385                .enable_all()
12386                .build()
12387                .unwrap()
12388                .block_on(async {
12389                    let mut child =
12390                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12391                    health_restart_child(
12392                        &spec,
12393                        &runtime,
12394                        &supervisor.registry,
12395                        &supervisor.process_liveness,
12396                        &snapshot,
12397                        &mut child,
12398                        SupervisorHealthStatus::Failing,
12399                        Some("health probe did not answer"),
12400                        1,
12401                    )
12402                    .await
12403                    .unwrap();
12404                    assert!(child.is_none());
12405                });
12406        });
12407        // The loopback barrier acknowledges entry into terminal recording by
12408        // health_restart_child's drain, then waits for the test's release byte.
12409        // Status and history are observed while that writer is held, not after
12410        // an elapsed-time guess about when the health action has completed.
12411        let mut reached = [0];
12412        barrier.read_exact(&mut reached).unwrap();
12413        assert_eq!(&reached, b"R");
12414        let before_status = module.status().unwrap();
12415        let before_history = module.terminal_history();
12416        barrier.write_all(b"X").unwrap();
12417        reap.join().unwrap();
12418        assert!(before_history.entries.is_empty());
12419        assert_ne!(
12420            before_status.state,
12421            ModuleState::Failed,
12422            "Failed was visible before its terminal record could be written"
12423        );
12424        assert!(
12425            before_status.process_alive,
12426            "process death was visible before its terminal record could be written"
12427        );
12428        let failed = module.status().unwrap();
12429        assert_eq!(failed.state, ModuleState::Failed);
12430        assert!(!failed.process_alive);
12431        let history = module.terminal_history();
12432        assert_eq!(history.entries.len(), 1);
12433        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12434        assert_eq!(
12435            history.entries[0].disposition_detail.as_deref(),
12436            Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12437        );
12438    }
12439
12440    #[test]
12441    fn observed_exits_publish_process_death_after_terminal_history() {
12442        use std::io::{Read, Write};
12443        for (kind, enabled, shutdown, wait_error, expected_state) in [
12444            (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12445            (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12446            (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12447            (
12448                ExitKind::DeliberateSeverance,
12449                true,
12450                false,
12451                false,
12452                ModuleState::Restarting,
12453            ),
12454            #[cfg(unix)]
12455            (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12456            (ExitKind::Crash, true, false, true, ModuleState::Failed),
12457        ] {
12458            let supervisor = Supervisor::new_for_test(
12459                Arc::new(Registry::default()),
12460                RestartPolicy::new(1, Duration::ZERO),
12461            );
12462            let runtime = supervisor.runtime_config();
12463            #[cfg(unix)]
12464            if shutdown {
12465                runtime.child_roster.close();
12466            }
12467            #[cfg(not(unix))]
12468            let _ = shutdown;
12469            let spec = windowed_crash_spec("exit-publication-order");
12470            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12471                ModuleState::Running,
12472                enabled,
12473            )));
12474            lock_snapshot(&snapshot).unwrap().process_alive = true;
12475            let reaped_snapshot = snapshot.clone();
12476            let history = runtime.terminal_ring.clone();
12477            let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12478            let reap = std::thread::spawn(move || {
12479                if wait_error {
12480                    super::fail_after_wait_error(
12481                        &spec.module_id,
12482                        &reaped_snapshot,
12483                        &runtime.terminal_ring,
12484                        &runtime.spawn_events,
12485                        "wait failed".into(),
12486                    );
12487                } else {
12488                    tokio::runtime::Builder::new_current_thread()
12489                        .enable_all()
12490                        .build()
12491                        .unwrap()
12492                        .block_on(on_child_exit(
12493                            &spec,
12494                            runtime.restart_policy,
12495                            &supervisor.registry,
12496                            &reaped_snapshot,
12497                            &runtime.terminal_ring,
12498                            &runtime.spawn_events,
12499                            &runtime.child_roster,
12500                            ExitReport {
12501                                kind,
12502                                code: Some(0),
12503                                signal: None,
12504                                at_ms: 1,
12505                            },
12506                        ));
12507                }
12508            });
12509            let mut reached = [0];
12510            barrier.read_exact(&mut reached).unwrap();
12511            assert_eq!(&reached, b"R");
12512            let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12513            let before_history = history.lock().unwrap().snapshot();
12514            barrier.write_all(b"X").unwrap();
12515            reap.join().unwrap();
12516            assert!(
12517                before_alive,
12518                "{kind:?}: death was published before recording"
12519            );
12520            assert!(before_history.entries.is_empty());
12521            assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12522            assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12523            assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12524        }
12525    }
12526
12527    #[tokio::test]
12528    async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12529        let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12530        let path = dir.join("terminals.jsonl");
12531        std::fs::create_dir(&path).unwrap();
12532        let supervisor = Supervisor::new_for_test(
12533            Arc::new(Registry::default()),
12534            RestartPolicy::new(0, Duration::ZERO),
12535        )
12536        .with_terminal_journal(path, "health-budget-journal-failure".into());
12537        let runtime = supervisor.runtime_config();
12538        let spec = ModuleSpec {
12539            program: fake_aft_stub_path(),
12540            protocol: ModuleProtocol::None,
12541            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12542            ..windowed_crash_spec("health-budget-journal-failure")
12543        };
12544        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12545        let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12546        health_restart_child(
12547            &spec,
12548            &runtime,
12549            &supervisor.registry,
12550            &supervisor.process_liveness,
12551            &snapshot,
12552            &mut child,
12553            SupervisorHealthStatus::Failing,
12554            None,
12555            1,
12556        )
12557        .await
12558        .unwrap();
12559        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12560        assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12561        let history = runtime
12562            .terminal_ring
12563            .lock()
12564            .unwrap()
12565            .durable_history(&spec.module_id);
12566        assert!(history.journal_write_failures > 0);
12567        assert_eq!(history.entries.len(), 1);
12568        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12569    }
12570
12571    #[test]
12572    fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12573        let supervisor = Supervisor::new_for_test(
12574            Arc::new(Registry::default()),
12575            RestartPolicy::new(0, Duration::ZERO),
12576        );
12577        let runtime = supervisor.runtime_config();
12578        let spec = ModuleSpec {
12579            program: fake_aft_stub_path(),
12580            protocol: ModuleProtocol::None,
12581            env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12582            ..windowed_crash_spec("health-budget-recording-panic")
12583        };
12584        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12585        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12586            tokio::runtime::Builder::new_current_thread()
12587                .enable_all()
12588                .build()
12589                .unwrap()
12590                .block_on(async {
12591                    let mut child =
12592                        Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12593                    runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12594                    health_restart_child(
12595                        &spec,
12596                        &runtime,
12597                        &supervisor.registry,
12598                        &supervisor.process_liveness,
12599                        &snapshot,
12600                        &mut child,
12601                        SupervisorHealthStatus::Failing,
12602                        None,
12603                        1,
12604                    )
12605                    .await
12606                    .unwrap();
12607                });
12608        }));
12609        let panic = result.expect_err("recording must still unwind");
12610        assert_eq!(
12611            panic.downcast_ref::<String>().map(String::as_str),
12612            Some("spawn event sequence exhausted")
12613        );
12614        let state = lock_snapshot(&snapshot).unwrap();
12615        assert_eq!(state.state, ModuleState::Failed);
12616        assert!(!state.process_alive);
12617        assert!(state.last_exit.is_some());
12618    }
12619
12620    #[tokio::test]
12621    async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12622        let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12623        let path = dir.join("terminals.jsonl");
12624        std::fs::create_dir(&path).unwrap();
12625        let supervisor = Supervisor::new_for_test(
12626            Arc::new(Registry::default()),
12627            RestartPolicy::new(0, Duration::ZERO),
12628        )
12629        .with_terminal_journal(path, "budget-journal-failure".into());
12630        let runtime = supervisor.runtime_config();
12631        let spec = windowed_crash_spec("budget-journal-failure");
12632        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12633        assert!(matches!(
12634            on_child_exit(
12635                &spec,
12636                runtime.restart_policy,
12637                &supervisor.registry,
12638                &snapshot,
12639                &runtime.terminal_ring,
12640                &runtime.spawn_events,
12641                &runtime.child_roster,
12642                crash_exit_report(1),
12643            )
12644            .await,
12645            NextAction::Stop { .. }
12646        ));
12647        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12648        let history = runtime
12649            .terminal_ring
12650            .lock()
12651            .unwrap()
12652            .durable_history(&spec.module_id);
12653        assert!(history.journal_write_failures > 0);
12654        assert_eq!(history.entries.len(), 1);
12655        assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12656    }
12657
12658    #[test]
12659    fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12660        let supervisor = Supervisor::new_for_test(
12661            Arc::new(Registry::default()),
12662            RestartPolicy::new(0, Duration::ZERO),
12663        );
12664        let runtime = supervisor.runtime_config();
12665        let spec = windowed_crash_spec("budget-recording-panic");
12666        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12667        runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12668        // Exhausting the event sequence makes emit_exited panic before the
12669        // terminal write, exercising publication on the recording unwind.
12670        runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12671        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12672            tokio::runtime::Builder::new_current_thread()
12673                .enable_all()
12674                .build()
12675                .unwrap()
12676                .block_on(on_child_exit(
12677                    &spec,
12678                    runtime.restart_policy,
12679                    &supervisor.registry,
12680                    &snapshot,
12681                    &runtime.terminal_ring,
12682                    &runtime.spawn_events,
12683                    &runtime.child_roster,
12684                    crash_exit_report(1),
12685                ))
12686        }));
12687        let panic = result.err().expect("recording must still unwind");
12688        assert_eq!(
12689            panic.downcast_ref::<String>().map(String::as_str),
12690            Some("spawn event sequence exhausted")
12691        );
12692        assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12693    }
12694
12695    /// A stop the supervisor itself requests still stops a protocol-none
12696    /// module, even though the child answers the SIGTERM with exit 0.
12697    #[cfg(unix)]
12698    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12699    async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12700        for disable in [false, true] {
12701            let label = if disable {
12702                "none-requested-disable"
12703            } else {
12704                "none-requested-stop"
12705            };
12706            let dir = cortexkit_test_support::ScratchDir::new(label);
12707            let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12708            let supervisor = Supervisor::new_for_test(
12709                Arc::new(Registry::default()),
12710                RestartPolicy::new(3, Duration::ZERO),
12711            );
12712            let module = supervisor.spawn(spec).unwrap();
12713            wait_for_file(&ready).await;
12714
12715            if disable {
12716                module.set_enabled(false).await.unwrap();
12717            } else {
12718                module.stop().await.unwrap();
12719            }
12720            assert!(
12721                marker.exists(),
12722                "{label}: the child must have left through its SIGTERM handler with exit 0"
12723            );
12724
12725            // Long enough for a zero-backoff respawn to have happened if the
12726            // exit had been treated as a crash.
12727            sleep(Duration::from_millis(500)).await;
12728            let status = module.status().unwrap();
12729            let expected = if disable {
12730                ModuleState::Disabled
12731            } else {
12732                ModuleState::Stopped
12733            };
12734            assert_eq!(status.state, expected, "{label}");
12735            assert_eq!(
12736                status.spawn_generation, 1,
12737                "{label}: respawned after a requested stop"
12738            );
12739            let history = module.terminal_history();
12740            assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12741            assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12742            assert_ne!(
12743                history.entries[0].disposition,
12744                TerminalDisposition::Restarting,
12745                "{label}"
12746            );
12747        }
12748    }
12749
12750    /// A subc-wire module that exits 0 on its own is still a stop: the
12751    /// protocol-none rule must not reach it.
12752    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12753    async fn subc_wire_clean_exit_is_still_a_stop() {
12754        let supervisor = Supervisor::new_for_test(
12755            Arc::new(Registry::default()),
12756            RestartPolicy::new(3, Duration::ZERO),
12757        );
12758        let module = supervisor
12759            .spawn(ModuleSpec {
12760                module_id: "wire-clean-exit".to_string(),
12761                program: fake_aft_stub_path(),
12762                args: Vec::new(),
12763                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12764                reserved: false,
12765                reserved_prefixes: Vec::new(),
12766                protocol: ModuleProtocol::Subc,
12767                overlap: Default::default(),
12768            })
12769            .unwrap();
12770
12771        let deadline = Instant::now() + Duration::from_secs(10);
12772        while module.terminal_history().entries.is_empty() {
12773            assert!(Instant::now() < deadline, "module never exited");
12774            sleep(Duration::from_millis(10)).await;
12775        }
12776        // Long enough for a zero-backoff respawn to have happened.
12777        sleep(Duration::from_millis(500)).await;
12778        let status = module.status().unwrap();
12779        assert_eq!(status.state, ModuleState::Stopped);
12780        assert_eq!(status.spawn_generation, 1);
12781        let history = module.terminal_history();
12782        assert_eq!(history.entries.len(), 1, "{history:?}");
12783        assert_eq!(history.entries[0].exit_code, Some(0));
12784        assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12785    }
12786
12787    #[cfg(unix)]
12788    #[tokio::test]
12789    async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12790        let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12791        let record = dir.join("live-children.json");
12792        let supervisor = Supervisor::new_for_test(
12793            Arc::new(Registry::default()),
12794            RestartPolicy::new(0, Duration::ZERO),
12795        );
12796        let mut runtime = supervisor.runtime_config();
12797        runtime.child_roster.record_to(record.clone());
12798        let gate = Arc::new(super::ReloadExitRecordGate::default());
12799        runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12800        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12801        let spec = ModuleSpec {
12802            module_id: "reload-exit-roster".into(),
12803            program: fake_aft_stub_path(),
12804            args: Vec::new(),
12805            env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12806            reserved: false,
12807            reserved_prefixes: Vec::new(),
12808            protocol: ModuleProtocol::Subc,
12809            overlap: Default::default(),
12810        };
12811        let mut child = None;
12812        let reload = super::finish_reload_child(
12813            &spec,
12814            &runtime,
12815            &supervisor.registry,
12816            &supervisor.process_liveness,
12817            &snapshot,
12818            &mut child,
12819        );
12820        tokio::pin!(reload);
12821        tokio::select! {
12822            result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12823            _ = gate.reached.notified() => {}
12824        }
12825        assert!(runtime
12826            .terminal_ring
12827            .lock()
12828            .unwrap()
12829            .snapshot()
12830            .entries
12831            .is_empty());
12832        assert_eq!(
12833            crate::live_children::read_record(&record).unwrap().len(),
12834            1,
12835            "shutdown must still wait for the reaped child until its terminal record exists"
12836        );
12837        runtime.child_roster.close();
12838        gate.resume.notify_one();
12839        assert!(reload.await.is_err());
12840        assert!(crate::live_children::read_record(&record)
12841            .unwrap()
12842            .is_empty());
12843        let history = runtime.terminal_ring.lock().unwrap().snapshot();
12844        assert_eq!(history.entries.len(), 1);
12845        assert_eq!(
12846            history.entries[0].disposition,
12847            TerminalDisposition::DaemonShutdown
12848        );
12849    }
12850
12851    /// Each restart-producing arm has its own state transition. Keeping their
12852    /// lifetime count assertions adjacent prevents a later new arm from silently
12853    /// spending budget without recording the historical restart.
12854    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12855    async fn every_restart_increment_path_advances_lifetime_count() {
12856        let supervisor = Supervisor::new_for_test(
12857            Arc::new(Registry::default()),
12858            RestartPolicy::new(1, Duration::ZERO),
12859        );
12860        let runtime = supervisor.runtime_config();
12861        let spec = ModuleSpec {
12862            module_id: "lifetime-increment-path".to_string(),
12863            program: PathBuf::from("/unused/lifetime-increment-path"),
12864            args: Vec::new(),
12865            env: Vec::new(),
12866            reserved: false,
12867            reserved_prefixes: Vec::new(),
12868            protocol: ModuleProtocol::Subc,
12869            overlap: Default::default(),
12870        };
12871
12872        let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12873        assert!(matches!(
12874            on_child_exit(
12875                &spec,
12876                runtime.restart_policy,
12877                &supervisor.registry,
12878                &crash_snapshot,
12879                &runtime.terminal_ring,
12880                &runtime.spawn_events,
12881                &runtime.child_roster,
12882                ExitReport {
12883                    kind: ExitKind::Crash,
12884                    code: Some(1),
12885                    signal: None,
12886                    at_ms: 1,
12887                },
12888            )
12889            .await,
12890            NextAction::Restart { schedule: _ }
12891        ));
12892        let (crash_restarts, crash_lifetime) = {
12893            let state = lock_snapshot(&crash_snapshot).unwrap();
12894            (state.crash_restarts.len(), state.lifetime_restarts)
12895        };
12896        assert_eq!(crash_restarts, 1);
12897        assert_eq!(crash_lifetime, 1);
12898
12899        let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12900        let mut health_child = None;
12901        assert!(matches!(
12902            health_restart_child(
12903                &spec,
12904                &runtime,
12905                &supervisor.registry,
12906                &supervisor.process_liveness,
12907                &health_snapshot,
12908                &mut health_child,
12909                SupervisorHealthStatus::Failing,
12910                None,
12911                2,
12912            )
12913            .await,
12914            Ok(())
12915        ));
12916        assert!(health_child.is_none());
12917        assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12918        assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12919        let (health_restarts, health_lifetime) = {
12920            let state = lock_snapshot(&health_snapshot).unwrap();
12921            (state.crash_restarts.len(), state.lifetime_restarts)
12922        };
12923        assert_eq!(health_restarts, 1);
12924        assert_eq!(health_lifetime, 1);
12925
12926        let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12927        let mut reload_child = None;
12928        assert!(matches!(
12929            handle_reload_spawn_failure(
12930                &spec,
12931                &runtime,
12932                &supervisor.process_liveness,
12933                &reload_snapshot,
12934                &mut reload_child,
12935                "forced reload spawn failure".to_string(),
12936            )
12937            .await,
12938            Err(SuperviseError::ReloadFailed { .. })
12939        ));
12940        let (reload_restarts, reload_lifetime) = {
12941            let state = lock_snapshot(&reload_snapshot).unwrap();
12942            (state.crash_restarts.len(), state.lifetime_restarts)
12943        };
12944        assert_eq!(reload_restarts, 1);
12945        assert_eq!(reload_lifetime, 1);
12946    }
12947
12948    #[tokio::test]
12949    async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12950        let supervisor = Supervisor::new_for_test(
12951            Arc::new(Registry::default()),
12952            RestartPolicy::new(3, Duration::ZERO),
12953        );
12954        let runtime = supervisor.runtime_config();
12955        let spec = ModuleSpec {
12956            module_id: "deliberately-severed".to_string(),
12957            program: PathBuf::from("/unused/deliberately-severed"),
12958            args: Vec::new(),
12959            env: Vec::new(),
12960            reserved: false,
12961            reserved_prefixes: Vec::new(),
12962            protocol: ModuleProtocol::Subc,
12963            overlap: Default::default(),
12964        };
12965        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12966        let process = ProcessIdentity {
12967            pid: 41,
12968            start_time: 101,
12969        };
12970        record_deliberate_severance(&snapshot, process).unwrap();
12971        let exit_report = apply_deliberate_severance_marker(
12972            &snapshot,
12973            Some(process),
12974            ExitReport {
12975                kind: ExitKind::Crash,
12976                code: Some(1),
12977                signal: None,
12978                at_ms: 1,
12979            },
12980        );
12981        assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12982
12983        assert!(matches!(
12984            on_child_exit(
12985                &spec,
12986                runtime.restart_policy,
12987                &supervisor.registry,
12988                &snapshot,
12989                &runtime.terminal_ring,
12990                &runtime.spawn_events,
12991                &runtime.child_roster,
12992                exit_report,
12993            )
12994            .await,
12995            NextAction::Restart { schedule: _ }
12996        ));
12997        let state = lock_snapshot(&snapshot).unwrap();
12998        assert_eq!(state.lifetime_restarts, 1);
12999        assert_eq!(state.crash_restarts.len(), 0);
13000    }
13001
13002    #[tokio::test]
13003    async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13004        let supervisor = Supervisor::new_for_test(
13005            Arc::new(Registry::default()),
13006            RestartPolicy::new(3, Duration::ZERO),
13007        );
13008        let runtime = supervisor.runtime_config();
13009        let spec = ModuleSpec {
13010            module_id: "genuine-crash".to_string(),
13011            program: PathBuf::from("/unused/genuine-crash"),
13012            args: Vec::new(),
13013            env: Vec::new(),
13014            reserved: false,
13015            reserved_prefixes: Vec::new(),
13016            protocol: ModuleProtocol::Subc,
13017            overlap: Default::default(),
13018        };
13019        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13020
13021        assert!(matches!(
13022            on_child_exit(
13023                &spec,
13024                runtime.restart_policy,
13025                &supervisor.registry,
13026                &snapshot,
13027                &runtime.terminal_ring,
13028                &runtime.spawn_events,
13029                &runtime.child_roster,
13030                ExitReport {
13031                    kind: ExitKind::Crash,
13032                    code: Some(1),
13033                    signal: None,
13034                    at_ms: 1,
13035                },
13036            )
13037            .await,
13038            NextAction::Restart { schedule: _ }
13039        ));
13040        let state = lock_snapshot(&snapshot).unwrap();
13041        assert_eq!(state.lifetime_restarts, 1);
13042        assert_eq!(state.crash_restarts.len(), 1);
13043    }
13044
13045    fn crash_exit_report(at_ms: u64) -> ExitReport {
13046        ExitReport {
13047            kind: ExitKind::Crash,
13048            code: Some(1),
13049            signal: None,
13050            at_ms,
13051        }
13052    }
13053
13054    fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13055        ModuleSpec {
13056            module_id: module_id.to_string(),
13057            program: PathBuf::from("/unused").join(module_id),
13058            args: Vec::new(),
13059            env: Vec::new(),
13060            reserved: false,
13061            reserved_prefixes: Vec::new(),
13062            protocol: ModuleProtocol::Subc,
13063            overlap: Default::default(),
13064        }
13065    }
13066
13067    /// A real crash loop still stops. Three crashes with nothing aging out spend
13068    /// a budget of two and the third respawn is refused, and both surfaces an
13069    /// operator has -- the log line and the retained terminal record -- name the
13070    /// window rather than only the cap, because `max_restarts=2` alone is what
13071    /// this budget used to mean.
13072    #[tokio::test]
13073    async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13074        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13075        let supervisor = Supervisor::new_for_test(
13076            Arc::new(Registry::default()),
13077            RestartPolicy::new(2, Duration::ZERO),
13078        );
13079        let runtime = supervisor.runtime_config();
13080        let spec = windowed_crash_spec("crash-loop-in-window");
13081        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13082
13083        for attempt in 1..=2 {
13084            assert!(
13085                matches!(
13086                    on_child_exit(
13087                        &spec,
13088                        runtime.restart_policy,
13089                        &supervisor.registry,
13090                        &snapshot,
13091                        &runtime.terminal_ring,
13092                        &runtime.spawn_events,
13093                        &runtime.child_roster,
13094                        crash_exit_report(attempt),
13095                    )
13096                    .await,
13097                    NextAction::Restart { schedule: _ }
13098                ),
13099                "crash {attempt} is inside the budget and must respawn"
13100            );
13101        }
13102
13103        assert!(matches!(
13104            on_child_exit(
13105                &spec,
13106                runtime.restart_policy,
13107                &supervisor.registry,
13108                &snapshot,
13109                &runtime.terminal_ring,
13110                &runtime.spawn_events,
13111                &runtime.child_roster,
13112                crash_exit_report(3),
13113            )
13114            .await,
13115            NextAction::Stop { .. }
13116        ));
13117
13118        {
13119            let state = lock_snapshot(&snapshot).unwrap();
13120            assert_eq!(state.state, ModuleState::Failed);
13121            assert_eq!(state.crash_restarts.len(), 2);
13122            assert_eq!(state.lifetime_restarts, 2);
13123        }
13124
13125        let history = runtime
13126            .terminal_ring
13127            .lock()
13128            .expect("terminal ring is not poisoned")
13129            .snapshot();
13130        let last = history
13131            .entries
13132            .last()
13133            .expect("the refused crash is retained");
13134        assert_eq!(last.disposition, TerminalDisposition::Failed);
13135        assert_eq!(
13136            last.disposition_detail.as_deref(),
13137            Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13138        );
13139
13140        let captured = crate::router::test_log::captured_logs(&logs);
13141        assert!(
13142            captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13143            "the stop must be logged with its window: {captured}"
13144        );
13145    }
13146
13147    /// The rate, stated as a test: three crashes where the first has aged past
13148    /// the window are two crashes as far as the budget is concerned, so the
13149    /// third respawn is allowed and the ring holds only the two recent ones.
13150    ///
13151    /// This is the case a lifetime counter got wrong -- and the case the daemon
13152    /// now hits routinely, since a module exits non-zero every time its
13153    /// connection to the daemon drops.
13154    #[tokio::test]
13155    async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13156        let supervisor = Supervisor::new_for_test(
13157            Arc::new(Registry::default()),
13158            RestartPolicy::new(2, Duration::ZERO),
13159        );
13160        let runtime = supervisor.runtime_config();
13161        let spec = windowed_crash_spec("crash-across-windows");
13162        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13163
13164        for attempt in 1..=2 {
13165            assert!(matches!(
13166                on_child_exit(
13167                    &spec,
13168                    runtime.restart_policy,
13169                    &supervisor.registry,
13170                    &snapshot,
13171                    &runtime.terminal_ring,
13172                    &runtime.spawn_events,
13173                    &runtime.child_roster,
13174                    crash_exit_report(attempt),
13175                )
13176                .await,
13177                NextAction::Restart { schedule: _ }
13178            ));
13179        }
13180
13181        // The oldest crash moves out of the window; nothing else about the
13182        // module changes.
13183        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13184            age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13185        })
13186        .unwrap();
13187
13188        assert!(
13189            matches!(
13190                on_child_exit(
13191                    &spec,
13192                    runtime.restart_policy,
13193                    &supervisor.registry,
13194                    &snapshot,
13195                    &runtime.terminal_ring,
13196                    &runtime.spawn_events,
13197                    &runtime.child_roster,
13198                    crash_exit_report(3),
13199                )
13200                .await,
13201                NextAction::Restart { schedule: _ }
13202            ),
13203            "a crash older than the window must not hold a budget slot"
13204        );
13205
13206        let state = lock_snapshot(&snapshot).unwrap();
13207        assert_eq!(state.state, ModuleState::Restarting);
13208        assert_eq!(
13209            state.crash_restarts.len(),
13210            2,
13211            "the aged instant is dropped and the new one takes its place"
13212        );
13213        assert_eq!(
13214            state.lifetime_restarts, 3,
13215            "the ledger counts every restart, including the ones the window forgot"
13216        );
13217    }
13218
13219    /// An operator restart hands the budget back whole, and the ledger keeps
13220    /// counting. Those are different questions -- "how close is this module to
13221    /// being stopped" and "how many times has it been replaced" -- and the
13222    /// operator action answers only the first.
13223    #[tokio::test]
13224    async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13225        let supervisor = Supervisor::new_for_test(
13226            Arc::new(Registry::default()),
13227            RestartPolicy::new(2, Duration::ZERO),
13228        );
13229        let runtime = supervisor.runtime_config();
13230        let spec = windowed_crash_spec("operator-cleared-budget");
13231        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13232
13233        for attempt in 1..=2 {
13234            assert!(matches!(
13235                on_child_exit(
13236                    &spec,
13237                    runtime.restart_policy,
13238                    &supervisor.registry,
13239                    &snapshot,
13240                    &runtime.terminal_ring,
13241                    &runtime.spawn_events,
13242                    &runtime.child_roster,
13243                    crash_exit_report(attempt),
13244                )
13245                .await,
13246                NextAction::Restart { schedule: _ }
13247            ));
13248        }
13249
13250        reset_restart_count(&snapshot, &spec.module_id).unwrap();
13251        {
13252            let state = lock_snapshot(&snapshot).unwrap();
13253            assert!(
13254                state.crash_restarts.is_empty(),
13255                "an operator restart returns the full budget"
13256            );
13257            assert_eq!(
13258                state.lifetime_restarts, 2,
13259                "clearing the budget must not unmake the crashes"
13260            );
13261        }
13262
13263        assert!(
13264            matches!(
13265                on_child_exit(
13266                    &spec,
13267                    runtime.restart_policy,
13268                    &supervisor.registry,
13269                    &snapshot,
13270                    &runtime.terminal_ring,
13271                    &runtime.spawn_events,
13272                    &runtime.child_roster,
13273                    crash_exit_report(3),
13274                )
13275                .await,
13276                NextAction::Restart { schedule: _ }
13277            ),
13278            "the cleared budget must be spendable again"
13279        );
13280        let state = lock_snapshot(&snapshot).unwrap();
13281        assert_eq!(state.crash_restarts.len(), 1);
13282        assert_eq!(state.lifetime_restarts, 3);
13283    }
13284
13285    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13286    async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13287        let severed = ProcessIdentity {
13288            pid: 41,
13289            start_time: 101,
13290        };
13291        let successor = ProcessIdentity {
13292            pid: 41,
13293            start_time: 202,
13294        };
13295        let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13296        update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13297            state.pid = Some(successor.pid);
13298            state.process_start_time = Some(successor.start_time);
13299        })
13300        .unwrap();
13301        assert!(!module.record_deliberate_severance(severed).unwrap());
13302
13303        let exit_report = apply_deliberate_severance_marker(
13304            &module.inner.snapshot,
13305            Some(successor),
13306            ExitReport {
13307                kind: ExitKind::Crash,
13308                code: Some(1),
13309                signal: None,
13310                at_ms: 1,
13311            },
13312        );
13313
13314        assert_eq!(exit_report.kind, ExitKind::Crash);
13315    }
13316
13317    #[tokio::test]
13318    async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13319        let registry = Registry::default();
13320        let supervisor = Supervisor::new_for_test(
13321            Arc::new(Registry::default()),
13322            RestartPolicy::new(3, Duration::ZERO),
13323        );
13324        let runtime = supervisor.runtime_config();
13325        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13326        let spec = ModuleSpec {
13327            module_id: "drain-deliberate-severance".to_string(),
13328            program: fake_aft_stub_path(),
13329            args: Vec::new(),
13330            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13331            reserved: false,
13332            reserved_prefixes: Vec::new(),
13333            protocol: ModuleProtocol::Subc,
13334            overlap: Default::default(),
13335        };
13336        let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13337        let process = ProcessIdentity {
13338            pid: 41,
13339            start_time: 101,
13340        };
13341        child.process_identity = Some(process);
13342        update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13343            state.pid = Some(process.pid);
13344            state.process_start_time = Some(process.start_time);
13345        })
13346        .unwrap();
13347        record_deliberate_severance(&snapshot, process).unwrap();
13348
13349        drain_child_to_state(
13350            &spec.module_id,
13351            spec.protocol,
13352            // The child exits on its own; no signal may change the exit this
13353            // test classifies.
13354            StopNotice::SentOverConnection,
13355            &registry,
13356            None,
13357            &snapshot,
13358            &runtime.terminal_ring,
13359            &runtime.spawn_events,
13360            child,
13361            Duration::from_secs(1),
13362            ModuleState::Stopped,
13363            Some(false),
13364        )
13365        .await
13366        .unwrap();
13367
13368        let state = lock_snapshot(&snapshot).unwrap();
13369        assert_eq!(
13370            state.last_exit.as_ref().map(|exit| exit.kind),
13371            Some(ExitKind::DeliberateSeverance)
13372        );
13373        assert_eq!(state.lifetime_restarts, 1);
13374        assert_eq!(state.crash_restarts.len(), 0);
13375        drop(state);
13376        let history = runtime.terminal_ring.lock().unwrap().snapshot();
13377        assert_eq!(
13378            history.entries[0].exit_kind,
13379            subc_control::TerminalExitKind::DeliberateSeverance
13380        );
13381    }
13382
13383    #[tokio::test]
13384    async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13385        let registry = Registry::default();
13386        let supervisor = Supervisor::new_for_test(
13387            Arc::new(Registry::default()),
13388            RestartPolicy::new(3, Duration::ZERO),
13389        );
13390        let runtime = supervisor.runtime_config();
13391        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13392        let spec = ModuleSpec {
13393            module_id: "ordinary-drain".to_string(),
13394            program: fake_aft_stub_path(),
13395            args: Vec::new(),
13396            env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13397            reserved: false,
13398            reserved_prefixes: Vec::new(),
13399            protocol: ModuleProtocol::Subc,
13400            overlap: Default::default(),
13401        };
13402        let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13403
13404        drain_child_to_state(
13405            &spec.module_id,
13406            spec.protocol,
13407            // The child exits on its own; no signal may change the exit this
13408            // test classifies.
13409            StopNotice::SentOverConnection,
13410            &registry,
13411            None,
13412            &snapshot,
13413            &runtime.terminal_ring,
13414            &runtime.spawn_events,
13415            child,
13416            Duration::from_secs(1),
13417            ModuleState::Stopped,
13418            Some(false),
13419        )
13420        .await
13421        .unwrap();
13422
13423        let state = lock_snapshot(&snapshot).unwrap();
13424        assert_eq!(
13425            state.last_exit.as_ref().map(|exit| exit.kind),
13426            Some(ExitKind::Crash)
13427        );
13428        assert_eq!(state.lifetime_restarts, 0);
13429        assert_eq!(state.crash_restarts.len(), 0);
13430    }
13431
13432    #[test]
13433    fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13434        // The server's generic fatal-routing branch only knows that the
13435        // connection failed; it does not know that the daemon deliberately
13436        // initiated a process-killing severance. Keep this seam explicit so a
13437        // future connection error path cannot silently reintroduce the stale
13438        // exemption that mislabels a later genuine crash.
13439        assert!(!include_str!("server.rs")
13440            .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13441    }
13442
13443    /// The `route.closed` `drained` value must be the quiescence wait's own
13444    /// measurement (`Ok`), never invented -- except on `Err`, where there is no
13445    /// measurement at all and `false` is the one honest constant. This is the exact
13446    /// logic `begin_forwarding_drain_with` now applies before sending `route.closed`
13447    /// on every return path, including the one that used to return early via `?`
13448    /// with `route.closing` already sent and no `route.closed` ever following.
13449    #[test]
13450    fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13451        assert!(drained_after_quiescence_wait(&Ok(true)));
13452        assert!(!drained_after_quiescence_wait(&Ok(false)));
13453        assert!(!drained_after_quiescence_wait(&Err(
13454            SuperviseError::StatePoisoned { module_id: None }
13455        )));
13456    }
13457
13458    /// `supervise_loop`'s `wait()`-error arm now calls `record_terminal` like every
13459    /// other exit path does, so a module whose child `wait()` itself errored (e.g.
13460    /// already reaped out-of-band) still leaves a terminal record rather than none
13461    /// at all. Triggering the real `wait()` I/O error from an integration test would
13462    /// need a genuine already-reaped-child race, which is OS-specific and not
13463    /// something this suite attempts elsewhere; this test instead verifies the
13464    /// record produced for that arm end-to-end through the real `TerminalRing`, and
13465    /// the call site itself is verified by inspection to sit in that exact arm.
13466    #[test]
13467    fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13468        let ring = Arc::new(Mutex::new(TerminalRing::new(
13469            TerminalRingConfig::default(),
13470            0,
13471        )));
13472        record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13473
13474        let snapshot = ring.lock().unwrap().snapshot();
13475        assert_eq!(snapshot.entries.len(), 1);
13476        let entry = &snapshot.entries[0];
13477        assert_eq!(entry.exit_code, None);
13478        assert_eq!(entry.exit_signal, None);
13479        assert_eq!(entry.disposition, TerminalDisposition::Failed);
13480    }
13481
13482    #[test]
13483    fn wait_error_exit_path_preserves_spawn_event_density() {
13484        let feed = super::SpawnEventFeed::default();
13485        feed.configure_incarnation("wait-error-density".to_string());
13486        feed.emit_spawned("wait-error", 41, 1);
13487        let ring = Arc::new(Mutex::new(TerminalRing::new(
13488            TerminalRingConfig::default(),
13489            0,
13490        )));
13491
13492        record_wait_error_terminal("wait-error", &ring, &feed);
13493        feed.emit_spawned("after-wait-error", 42, 2);
13494
13495        let state = feed.0.lock().unwrap();
13496        let sequences = state
13497            .events
13498            .iter()
13499            .map(|event| event.cursor.seq)
13500            .collect::<Vec<_>>();
13501        assert_eq!(sequences, vec![1, 2, 3]);
13502        assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13503        assert_eq!(state.events[1].exit_code, None);
13504        assert_eq!(state.events[1].exit_signal, None);
13505    }
13506
13507    /// Pins the report's `kind` too: the wait-error arm treats an unwaitable child
13508    /// as a crash (matching `fail_snapshot`'s `Failed` disposition for this arm),
13509    /// not a clean exit it never actually observed.
13510    #[test]
13511    fn wait_error_exit_report_is_classified_as_a_crash() {
13512        assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13513    }
13514}
13515
13516#[cfg(all(test, unix))]
13517mod restart_ordering_tests {
13518    use super::*;
13519    use std::io::Write;
13520    use tokio::io::AsyncReadExt;
13521
13522    #[test]
13523    #[ignore = "child process fixture"]
13524    fn serving_child() {
13525        let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13526            return;
13527        };
13528        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13529        std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13530        for stream in listener.incoming() {
13531            writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13532        }
13533    }
13534
13535    async fn restart_before_publication(requests: usize) {
13536        let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13537        let address_file = home.path().join("address");
13538        let spec = ModuleSpec {
13539            module_id: "restart-serving-child".into(),
13540            program: std::env::current_exe().unwrap(),
13541            args: vec![
13542                "--exact".into(),
13543                "supervise::restart_ordering_tests::serving_child".into(),
13544                "--ignored".into(),
13545                "--nocapture".into(),
13546            ],
13547            env: vec![(
13548                "RESTART_TEST_ADDRESS_FILE".into(),
13549                address_file.display().to_string(),
13550            )],
13551            reserved: false,
13552            reserved_prefixes: vec![],
13553            protocol: ModuleProtocol::None,
13554            overlap: Default::default(),
13555        };
13556        let mut supervisor =
13557            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13558        let (spawned_tx, spawned_rx) = oneshot::channel();
13559        let spawned_tx = Mutex::new(Some(spawned_tx));
13560        let (release_tx, release_rx) = std::sync::mpsc::channel();
13561        let release_rx = Mutex::new(release_rx);
13562        // Hold publication, not the child: it must answer a real request before
13563        // the restart enters the queue, even though status still has generation 0.
13564        supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13565            spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13566            release_rx
13567                .lock()
13568                .unwrap()
13569                .recv_timeout(Duration::from_secs(20))
13570                .unwrap();
13571        })));
13572        let registered = supervisor
13573            .register_configured_with_health(
13574                spec.clone(),
13575                true,
13576                HealthConfig::default(),
13577                None,
13578                RestartPolicy::new(3, Duration::ZERO),
13579            )
13580            .unwrap();
13581        let module = registered.module.clone();
13582        let startup = tokio::task::spawn_blocking(move || {
13583            supervisor.start_registered_configured(registered, None)
13584        });
13585        spawned_rx.await.unwrap();
13586        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13587            let address = loop {
13588                if let Ok(address) = std::fs::read_to_string(&address_file) {
13589                    if !address.is_empty() {
13590                        break address;
13591                    }
13592                }
13593                tokio::time::sleep(Duration::from_millis(1)).await;
13594            };
13595            let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13596            let mut reply = String::new();
13597            stream.read_to_string(&mut reply).await.unwrap();
13598            reply.trim().parse::<u32>().unwrap()
13599        })
13600        .await
13601        .expect("child must serve before generation publication");
13602        assert_eq!(
13603            lock_snapshot(&module.inner.snapshot)
13604                .unwrap()
13605                .spawn_generation,
13606            0
13607        );
13608        let mut replies = Vec::new();
13609        for _ in 0..requests {
13610            let mut restart = Box::pin(module.restart(Some(0)));
13611            // Poll the public API through queue admission, leaving only its ACK
13612            // pending. The monitor cannot consume the command until released.
13613            std::future::poll_fn(|cx| {
13614                assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13615                std::task::Poll::Ready(())
13616            })
13617            .await;
13618            replies.push(restart);
13619        }
13620        release_tx.send(()).unwrap();
13621        startup.await.unwrap().unwrap();
13622        for reply in replies {
13623            reply.await.unwrap();
13624        }
13625        // Restart schedules its replacement after completing the handler. Wait
13626        // for that separate actor turn before placing the command barrier.
13627        let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13628            loop {
13629                if lock_snapshot(&module.inner.snapshot)
13630                    .unwrap()
13631                    .spawn_generation
13632                    > 1
13633                {
13634                    break;
13635                }
13636                tokio::time::sleep(Duration::from_millis(1)).await;
13637            }
13638        })
13639        .await;
13640        // Commands are handled serially. The update-spec reply proves every
13641        // restart handler has finished before we inspect the snapshot; restart's
13642        // own ACK only acknowledges initiation, not replacement completion.
13643        module.update_spec_for_test(spec).await.unwrap();
13644        let (generation, pid) = {
13645            let state = lock_snapshot(&module.inner.snapshot).unwrap();
13646            (state.spawn_generation, state.pid)
13647        };
13648        module.stop().await.unwrap();
13649        replacement.expect("restart must publish a replacement for the already-serving child");
13650        assert_ne!(
13651            pid,
13652            Some(old_pid),
13653            "a serving child predating the restart must be replaced"
13654        );
13655        assert_eq!(generation, 2, "queued restarts must share one replacement");
13656    }
13657
13658    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13659    async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13660        restart_before_publication(1).await;
13661    }
13662
13663    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13664    async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13665        restart_before_publication(2).await;
13666    }
13667
13668    #[tokio::test]
13669    async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13670        let supervisor =
13671            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13672        let registered = supervisor
13673            .register_configured_with_health(
13674                ModuleSpec {
13675                    module_id: "queued-before-replacement".into(),
13676                    // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on
13677                    // macOS a module whose running image is not the configured
13678                    // program is refused once its exec is confirmed (see
13679                    // `record_write_stall_tests::sleeping_spec`).
13680                    program: "/bin/sleep".into(),
13681                    args: vec!["600".into()],
13682                    env: vec![],
13683                    reserved: false,
13684                    reserved_prefixes: vec![],
13685                    protocol: ModuleProtocol::None,
13686                    overlap: Default::default(),
13687                },
13688                true,
13689                HealthConfig::default(),
13690                None,
13691                RestartPolicy::default(),
13692            )
13693            .unwrap();
13694        let RegisteredConfiguredModule {
13695            mut spec,
13696            mut runtime,
13697            module,
13698            mut commands,
13699        } = registered;
13700        let snapshot = &module.inner.snapshot;
13701        let mut restart = Box::pin(module.restart(Some(0)));
13702        std::future::poll_fn(|cx| {
13703            assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13704            std::task::Poll::Ready(())
13705        })
13706        .await;
13707        // Drive the actor explicitly so a replacement is definitely spawned
13708        // after receipt, but before this queued command is dequeued.
13709        let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13710        let replacement_pid = child.as_ref().unwrap().id();
13711        let command = commands.recv().await.unwrap();
13712        assert!(
13713            handle_supervisor_command(
13714                command,
13715                &mut spec,
13716                &mut runtime,
13717                &supervisor.registry,
13718                &supervisor.process_liveness,
13719                snapshot,
13720                &mut child,
13721                &mut commands,
13722                &mut VecDeque::new(),
13723            )
13724            .await
13725        );
13726        restart.await.unwrap();
13727        let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13728        let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13729        if let Some(mut child) = child {
13730            child.start_kill().unwrap();
13731            child.wait().await.unwrap();
13732        }
13733        assert_eq!(
13734            kept_pid, replacement_pid,
13735            "a newer replacement must survive"
13736        );
13737        assert!(
13738            !pending,
13739            "a satisfied restart must not schedule another spawn"
13740        );
13741    }
13742}
13743
13744#[cfg(all(test, unix))]
13745mod health_event_tests {
13746    use super::*;
13747    use cortexkit_test_support::ScratchDir;
13748    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13749
13750    struct Actor {
13751        module: SupervisedModule,
13752        registry: Arc<Registry>,
13753        forwarding: Arc<ForwardingTable>,
13754        spec: ModuleSpec,
13755        health: HealthConfig,
13756        rx: mpsc::Receiver<crate::router::OutboundFrame>,
13757        _home: ScratchDir,
13758    }
13759
13760    async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13761        // OS exec and socket readiness are real I/O. Stay runnable while waiting
13762        // for an observed condition, rather than letting paused time auto-advance
13763        // deadlines or assuming a fixed number of yields finishes the work.
13764        let deadline = std::time::Instant::now() + Duration::from_secs(10);
13765        let tick = Instant::now();
13766        loop {
13767            if let Some(value) = observe() {
13768                assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13769                return value;
13770            }
13771            assert!(
13772                std::time::Instant::now() < deadline,
13773                "timed out waiting for {reason}"
13774            );
13775            tokio::task::yield_now().await;
13776        }
13777    }
13778
13779    impl Actor {
13780        async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13781            let home = ScratchDir::new("health-events");
13782            let registry = Arc::new(Registry::default());
13783            let forwarding = Arc::new(ForwardingTable::default());
13784            let health = HealthConfig {
13785                cadence,
13786                deadline: Duration::from_secs(3600),
13787                ..HealthConfig::default()
13788            };
13789            let supervisor =
13790                Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13791                    .with_forwarding(forwarding.clone())
13792                    .with_health_config(health.clone());
13793            let spec = ModuleSpec {
13794                module_id: "event-health-child".into(),
13795                // `/bin/sleep` itself, not `/bin/sh -c "exec sleep"`: on macOS
13796                // the privacy trampoline refuses a module whose running image
13797                // is not the configured program, and `start` below waits for
13798                // the process to be confirmed (`reported_pid`). The shell form
13799                // passed only while the image check ran before the shell had
13800                // replaced itself with `sleep` (see
13801                // `record_write_stall_tests::sleeping_spec`).
13802                program: PathBuf::from("/bin/sleep"),
13803                // The process stays alive but has no real bus peer. Registrations
13804                // below exercise the same registry writes as accepted HELLOs.
13805                args: vec!["600".into()],
13806                env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13807                    .into_iter()
13808                    .map(|key| (key.into(), home.path().display().to_string()))
13809                    .collect(),
13810                reserved: false,
13811                reserved_prefixes: vec![],
13812                protocol,
13813                overlap: Default::default(),
13814            };
13815            let module = supervisor.spawn(spec.clone()).unwrap();
13816            let (_, rx) = mpsc::channel(8);
13817            let actor = Self {
13818                module,
13819                registry,
13820                forwarding,
13821                spec,
13822                health,
13823                rx,
13824                _home: home,
13825            };
13826            actor
13827                .wait_for_select(|checkpoint| checkpoint.generation > 0)
13828                .await;
13829            actor
13830        }
13831
13832        fn turns(&self) -> u64 {
13833            lock_snapshot(&self.module.inner.snapshot)
13834                .unwrap()
13835                .actor_turns
13836        }
13837
13838        async fn wait_for_select(
13839            &self,
13840            expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13841        ) -> ActorSelectCheckpoint {
13842            wait_for("actor select checkpoint", || {
13843                let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13844                state.actor_select.filter(|checkpoint| {
13845                    checkpoint.generation == state.spawn_generation
13846                        && state.state == ModuleState::Running
13847                        && state.process_alive
13848                        && state.reported_pid().is_some()
13849                        && expected(checkpoint)
13850                })
13851            })
13852            .await
13853        }
13854
13855        fn hello(&mut self, connection: u64, health: bool) {
13856            let manifest =
13857                subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13858                    .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13859                    .build();
13860            let connection = ConnectionId::new(connection);
13861            let (tx, rx) = mpsc::channel(8);
13862            self.rx = rx;
13863            self.forwarding
13864                .register_module_connection(
13865                    connection,
13866                    self.spec.module_id.clone(),
13867                    subc_protocol::PROTOCOL_VERSION,
13868                    Concurrency::ModuleManaged,
13869                    FrameSink::new(tx),
13870                )
13871                .unwrap();
13872            self.registry
13873                .register_with_control_ops(
13874                    manifest,
13875                    subc_protocol::PROTOCOL_VERSION,
13876                    connection,
13877                    if health {
13878                        vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13879                    } else {
13880                        vec![]
13881                    },
13882                )
13883                .unwrap();
13884        }
13885
13886        async fn probe(&mut self) -> crate::router::OutboundFrame {
13887            let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13888                Ok(frame) => Some(frame),
13889                Err(mpsc::error::TryRecvError::Empty) => None,
13890                Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13891            })
13892            .await;
13893            let body: Value = serde_json::from_slice(&frame.body).unwrap();
13894            assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13895            frame
13896        }
13897
13898        fn no_probe(&mut self) {
13899            assert!(matches!(
13900                self.rx.try_recv(),
13901                Err(mpsc::error::TryRecvError::Empty)
13902            ));
13903        }
13904
13905        fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13906            self.forwarding
13907                .complete_module_control_rpc(
13908                    ConnectionId::new(connection),
13909                    frame.header.corr,
13910                    Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13911                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13912                        status: HealthStatus::Ok,
13913                        detail: None,
13914                        metrics: None,
13915                    }),
13916                )
13917                .unwrap();
13918        }
13919    }
13920
13921    #[tokio::test(start_paused = true)]
13922    async fn no_health_children_park_for_sixty_seconds() {
13923        let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13924        let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13925        let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13926        unadvertised.hello(1, false);
13927        // Include any one-off registration wake in the budget; receiving a
13928        // no-health HELLO must not turn a parked child into a polling child.
13929        // Advance in 10 ms steps so the old poll really executes ~6,000 turns;
13930        // a single 60 s jump would only observe one expired timer.
13931        for _ in 0..6000 {
13932            tokio::time::advance(Duration::from_millis(10)).await;
13933            tokio::task::yield_now().await;
13934        }
13935        for actor in [&none, &unregistered, &unadvertised] {
13936            assert!(
13937                actor.turns() <= 5,
13938                "no-health actor woke {} times",
13939                actor.turns()
13940            );
13941            assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13942        }
13943    }
13944
13945    #[tokio::test(start_paused = true)]
13946    async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13947        // With a zero cadence the first probe is due at once, so this measures
13948        // only how fast the registration notification starts health probing,
13949        // not the normal 30-33 s wait before a first probe (checked below).
13950        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13951        let tick = Instant::now();
13952        actor.hello(1, true);
13953        actor.probe().await;
13954        assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13955    }
13956
13957    #[tokio::test(start_paused = true)]
13958    async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13959        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13960        let hello_tick = Instant::now();
13961        actor.hello(1, true);
13962        let armed = actor
13963            .wait_for_select(|checkpoint| {
13964                checkpoint.registered_connection == Some(ConnectionId::new(1))
13965                    && checkpoint.next_probe_at.is_some()
13966            })
13967            .await;
13968        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13969        assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13970        let first_deadline = hello_tick + delay;
13971        assert_eq!(armed.next_probe_at, Some(first_deadline));
13972        tokio::time::advance(Duration::from_secs(29)).await;
13973        let turns = actor.turns();
13974        actor
13975            .registry
13976            .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13977            .unwrap();
13978        let updated = actor
13979            .wait_for_select(|checkpoint| checkpoint.turn > turns)
13980            .await;
13981        assert_eq!(
13982            updated.next_probe_at,
13983            Some(first_deadline),
13984            "catalog update must not reset cadence"
13985        );
13986        actor.no_probe();
13987        tokio::time::advance(first_deadline - Instant::now()).await;
13988        let frame = actor.probe().await;
13989        actor.answer(1, frame);
13990        let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13991        let next_deadline = Instant::now() + next;
13992        let rearmed = actor
13993            .wait_for_select(|checkpoint| {
13994                checkpoint
13995                    .next_probe_at
13996                    .is_some_and(|deadline| deadline > first_deadline)
13997            })
13998            .await;
13999        assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14000        tokio::time::advance(next - Duration::from_nanos(1)).await;
14001        actor.no_probe();
14002        tokio::time::advance(Duration::from_nanos(1)).await;
14003        actor.probe().await;
14004    }
14005
14006    #[tokio::test(start_paused = true)]
14007    async fn disconnect_disarms_and_reconnect_rearms_health() {
14008        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14009        actor.hello(1, true);
14010        actor
14011            .wait_for_select(|checkpoint| {
14012                checkpoint.registered_connection == Some(ConnectionId::new(1))
14013                    && checkpoint.next_probe_at.is_some()
14014            })
14015            .await;
14016        let turns = actor.turns();
14017        let tick = Instant::now();
14018        actor
14019            .registry
14020            .deregister_connection(ConnectionId::new(1))
14021            .unwrap();
14022        let disarmed = actor
14023            .wait_for_select(|checkpoint| {
14024                checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14025            })
14026            .await;
14027        assert_eq!(disarmed.next_probe_at, None);
14028        assert_eq!(disarmed.wake_after, None);
14029        assert_eq!(
14030            Instant::now(),
14031            tick,
14032            "disconnect must disarm in the notification tick"
14033        );
14034        tokio::time::advance(Duration::from_secs(60)).await;
14035        actor.no_probe();
14036        let reconnect_tick = Instant::now();
14037        actor.hello(2, true);
14038        let rearmed = actor
14039            .wait_for_select(|checkpoint| {
14040                checkpoint.registered_connection == Some(ConnectionId::new(2))
14041                    && checkpoint.next_probe_at.is_some()
14042            })
14043            .await;
14044        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14045        assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14046        tokio::time::advance(delay).await;
14047        actor.probe().await;
14048    }
14049
14050    #[tokio::test(start_paused = true)]
14051    async fn reregistration_without_health_disarms_the_old_schedule() {
14052        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14053        actor.hello(1, true);
14054        actor
14055            .wait_for_select(|checkpoint| {
14056                checkpoint.registered_connection == Some(ConnectionId::new(1))
14057                    && checkpoint.next_probe_at.is_some()
14058            })
14059            .await;
14060        actor
14061            .registry
14062            .deregister_connection(ConnectionId::new(1))
14063            .unwrap();
14064        actor.hello(2, false);
14065        let disarmed = actor
14066            .wait_for_select(|checkpoint| {
14067                checkpoint.registered_connection == Some(ConnectionId::new(2))
14068            })
14069            .await;
14070        assert_eq!(disarmed.next_probe_at, None);
14071        assert_eq!(disarmed.wake_after, None);
14072        tokio::time::advance(Duration::from_secs(60)).await;
14073        actor.no_probe();
14074    }
14075
14076    #[tokio::test(start_paused = true)]
14077    async fn restart_waits_for_its_own_hello_before_rearming_health() {
14078        let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14079        actor.hello(1, true);
14080        let armed = actor
14081            .wait_for_select(|checkpoint| {
14082                checkpoint.registered_connection == Some(ConnectionId::new(1))
14083                    && checkpoint.next_probe_at.is_some()
14084            })
14085            .await;
14086        // Simulate the old process's connection closing before the restart
14087        // tears the process down. The replacement process has started but has
14088        // not sent its HELLO, so it is not registered.
14089        actor
14090            .registry
14091            .deregister_connection(ConnectionId::new(1))
14092            .unwrap();
14093        actor.module.restart(Some(0)).await.unwrap();
14094        let parked = actor
14095            .wait_for_select(|checkpoint| {
14096                checkpoint.generation > armed.generation
14097                    && checkpoint.registered_connection.is_none()
14098                    && checkpoint.next_probe_at.is_none()
14099            })
14100            .await;
14101        assert_eq!(parked.wake_after, None);
14102        actor.rx.close();
14103        // Drain the stop notice the restart queued for the old process's connection,
14104        // so it isn't mistaken for traffic to the replacement.
14105        while actor.rx.try_recv().is_ok() {}
14106        let turns = actor.turns();
14107        tokio::time::advance(Duration::from_secs(60)).await;
14108        let after = actor.turns();
14109        eprintln!(
14110            "restart park turns: before={turns} after={after} delta={}",
14111            after - turns
14112        );
14113        assert_eq!(after, turns, "replacement must park before HELLO");
14114        let hello_tick = Instant::now();
14115        actor.hello(2, true);
14116        let rearmed = actor
14117            .wait_for_select(|checkpoint| {
14118                checkpoint.registered_connection == Some(ConnectionId::new(2))
14119                    && checkpoint.next_probe_at.is_some()
14120            })
14121            .await;
14122        let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14123        assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14124        tokio::time::advance(delay).await;
14125        actor.probe().await;
14126    }
14127
14128    #[tokio::test(start_paused = true)]
14129    async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14130        use tokio::{
14131            io::{AsyncReadExt, AsyncWriteExt},
14132            net::TcpListener,
14133        };
14134        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14135        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14136        actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14137        let (started_tx, mut started_rx) = mpsc::channel(8);
14138        let peer = tokio::spawn(async move {
14139            loop {
14140                let (mut socket, _) = listener.accept().await.unwrap();
14141                let mut request = [0; 1024];
14142                assert!(socket.read(&mut request).await.unwrap() > 0);
14143                started_tx.send(Instant::now()).await.unwrap();
14144                socket
14145                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14146                    .await
14147                    .unwrap();
14148            }
14149        });
14150        actor
14151            .module
14152            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14153            .await
14154            .unwrap();
14155        let first_tick = Instant::now();
14156        let first = actor
14157            .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14158            .await;
14159        assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14160        assert!(started_rx.try_recv().is_err());
14161
14162        // Restart just before the old deadline: reusing it would give the new
14163        // process only one second to start, rather than a full cadence.
14164        tokio::time::advance(
14165            first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14166        )
14167        .await;
14168        actor.module.restart(Some(0)).await.unwrap();
14169        let spawn_tick = Instant::now();
14170        let replacement = actor
14171            .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14172            .await;
14173        let deadline = replacement.next_probe_at.unwrap();
14174        assert!(
14175            deadline >= spawn_tick + actor.health.cadence,
14176            "a replacement must not inherit its predecessor's probe deadline"
14177        );
14178        tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14179        assert!(
14180            started_rx.try_recv().is_err(),
14181            "no HTTP probe during startup grace"
14182        );
14183        tokio::time::advance(deadline - Instant::now()).await;
14184        let landed = wait_for("first replacement HTTP probe", || {
14185            started_rx.try_recv().ok()
14186        })
14187        .await;
14188        assert!(landed >= spawn_tick + actor.health.cadence);
14189        wait_for("healthy replacement", || {
14190            (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14191                .then_some(())
14192        })
14193        .await;
14194        peer.abort();
14195    }
14196
14197    #[tokio::test(start_paused = true)]
14198    async fn rescan_adds_and_removes_http_health_without_polling() {
14199        use tokio::{
14200            io::{AsyncReadExt, AsyncWriteExt},
14201            net::TcpListener,
14202        };
14203        let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14204        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14205        actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14206        // One millisecond separates successive probes without a polling delay.
14207        actor.health.cadence = Duration::from_millis(1);
14208        let (started_tx, mut started_rx) = mpsc::channel(8);
14209        let peer = tokio::spawn(async move {
14210            loop {
14211                let (mut socket, _) = listener.accept().await.unwrap();
14212                let mut request = [0; 1024];
14213                assert!(socket.read(&mut request).await.unwrap() > 0);
14214                started_tx.send(()).await.unwrap();
14215                socket
14216                    .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14217                    .await
14218                    .unwrap();
14219            }
14220        });
14221        let tick = Instant::now();
14222        let turns = actor.turns();
14223        actor
14224            .module
14225            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14226            .await
14227            .unwrap();
14228        let armed = actor
14229            .wait_for_select(|checkpoint| {
14230                checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14231            })
14232            .await;
14233        assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14234        assert_eq!(Instant::now(), tick);
14235        tokio::time::advance(Duration::from_millis(1)).await;
14236        let mut started = false;
14237        wait_for("successful HTTP probe", || {
14238            started |= started_rx.try_recv().is_ok();
14239            (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14240                .then_some(())
14241        })
14242        .await;
14243        assert!(started, "added HTTP check must start at its first deadline");
14244        assert_eq!(
14245            actor.module.status().unwrap().health.status,
14246            SupervisorHealthStatus::Ok
14247        );
14248        assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14249        actor.health.http = None;
14250        let turns = actor.turns();
14251        actor
14252            .module
14253            .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14254            .await
14255            .unwrap();
14256        let parked = actor
14257            .wait_for_select(|checkpoint| {
14258                checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14259            })
14260            .await;
14261        assert_eq!(parked.wake_after, None);
14262        let turns = actor.turns();
14263        tokio::time::advance(Duration::from_secs(60)).await;
14264        assert_eq!(
14265            actor.turns(),
14266            turns,
14267            "removed HTTP check must park the actor"
14268        );
14269        assert!(
14270            started_rx.try_recv().is_err(),
14271            "removed HTTP check must stay disarmed"
14272        );
14273        assert_eq!(
14274            actor.module.status().unwrap().health,
14275            ModuleHealthStatus::default()
14276        );
14277        peer.abort();
14278    }
14279}
14280
14281#[cfg(test)]
14282mod drain_busy_reprobe_tests {
14283    //! The drain's busy-gauge wait, driven against an in-process module
14284    //! connection whose health answers the test scripts. Time is paused, so
14285    //! every elapsed duration below is exact virtual time.
14286    use super::*;
14287    use std::collections::BTreeMap;
14288    use std::sync::Mutex as StdMutex;
14289    use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14290
14291    const MODULE_ID: &str = "drain-reprobe-module";
14292    const CONNECTION: u64 = 7;
14293    const GAUGE: &str = "runs_in_flight";
14294    /// The production default for both the health cadence and the drain
14295    /// ceiling, which is the pairing that used to cost a full ceiling.
14296    const CADENCE: Duration = Duration::from_secs(30);
14297    const CEILING: Duration = Duration::from_secs(30);
14298
14299    /// How the scripted module answers each `health.check`.
14300    #[derive(Clone, Copy)]
14301    enum Answer {
14302        /// Report the gauge busy until this instant, quiet from then on, after
14303        /// taking `latency` to answer.
14304        BusyUntil { until: Instant, latency: Duration },
14305        /// Answer at once, with metrics that leave the declared gauge out.
14306        OmitsGauge,
14307        /// Never answer.
14308        Silent,
14309    }
14310
14311    struct Drain {
14312        forwarding: Arc<ForwardingTable>,
14313        runtime: SupervisorRuntimeConfig,
14314        endpoint: crate::ModuleEndpointId,
14315        /// When each probe reached the module.
14316        probes: Arc<StdMutex<Vec<Instant>>>,
14317        responder: tokio::task::JoinHandle<()>,
14318    }
14319
14320    impl Drain {
14321        fn start(answer: Answer) -> Self {
14322            let registry = Arc::new(Registry::default());
14323            let forwarding = Arc::new(ForwardingTable::default());
14324            let health = HealthConfig {
14325                cadence: CADENCE,
14326                ..HealthConfig::default()
14327            };
14328            let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14329                .with_forwarding(forwarding.clone())
14330                .with_health_config(health)
14331                .runtime_config();
14332            let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14333            let endpoint = forwarding
14334                .register_module_connection(
14335                    ConnectionId::new(CONNECTION),
14336                    MODULE_ID.to_string(),
14337                    subc_protocol::PROTOCOL_VERSION,
14338                    Concurrency::ModuleManaged,
14339                    FrameSink::new(tx),
14340                )
14341                .unwrap();
14342            let probes = Arc::new(StdMutex::new(Vec::new()));
14343            let responder = tokio::spawn({
14344                let forwarding = forwarding.clone();
14345                let probes = probes.clone();
14346                async move {
14347                    while let Some(frame) = rx.recv().await {
14348                        let body: Value = serde_json::from_slice(&frame.body).unwrap();
14349                        assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14350                        probes.lock().unwrap().push(Instant::now());
14351                        let metrics = match answer {
14352                            Answer::BusyUntil { until, latency } => {
14353                                sleep(latency).await;
14354                                let busy = u64::from(Instant::now() < until);
14355                                serde_json::json!({ GAUGE: busy })
14356                            }
14357                            Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14358                            Answer::Silent => continue,
14359                        };
14360                        // A probe abandoned at its deadline has no RPC left to
14361                        // complete, so a late answer may be refused.
14362                        let _ = forwarding.complete_module_control_rpc(
14363                            ConnectionId::new(CONNECTION),
14364                            frame.header.corr,
14365                            Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14366                            ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14367                                status: HealthStatus::Ok,
14368                                detail: None,
14369                                metrics: Some(metrics),
14370                            }),
14371                        );
14372                    }
14373                }
14374            });
14375            Self {
14376                forwarding,
14377                runtime,
14378                endpoint,
14379                probes,
14380                responder,
14381            }
14382        }
14383
14384        /// Run one by-id drain wait with `ceiling`, returning its verdict and
14385        /// how long it took.
14386        async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14387            let (drained, elapsed, _) = self.run_traced(ceiling).await;
14388            (drained, elapsed)
14389        }
14390
14391        /// [`Self::run`], also returning what the drain's probes read.
14392        async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14393            let started = Instant::now();
14394            let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14395            let drained = wait_for_forwarding_quiescence(
14396                &self.forwarding,
14397                MODULE_ID,
14398                &self.runtime,
14399                self.endpoint,
14400                started + ceiling,
14401                &[GAUGE.to_string()],
14402                DrainScope::Active,
14403                &mut trace,
14404            )
14405            .await
14406            .unwrap();
14407            (drained, started.elapsed(), trace)
14408        }
14409
14410        fn probes(&self) -> Vec<Instant> {
14411            self.probes.lock().unwrap().clone()
14412        }
14413    }
14414
14415    impl Drop for Drain {
14416        fn drop(&mut self) {
14417            self.responder.abort();
14418        }
14419    }
14420
14421    #[tokio::test(start_paused = true)]
14422    async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14423        let drain = Drain::start(Answer::BusyUntil {
14424            until: Instant::now() + Duration::from_secs(1),
14425            latency: Duration::ZERO,
14426        });
14427        let (drained, elapsed) = drain.run(CEILING).await;
14428        assert!(
14429            drained,
14430            "gauges that read zero during the drain must let it finish drained, \
14431             not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14432        );
14433        assert!(
14434            elapsed < Duration::from_secs(3),
14435            "drain took {elapsed:?} after the gauges went quiet at 1s"
14436        );
14437        let probes = drain.probes();
14438        assert!(
14439            probes.len() >= 2,
14440            "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14441        );
14442    }
14443
14444    /// The same shape in real time on a multi-threaded runtime: the first
14445    /// probe is answered busy at once, within a millisecond of the drain
14446    /// starting, and later probes read idle. The drain must keep probing
14447    /// after that first instant answer and end soon after the gauge falls.
14448    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14449    async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14450        let drain = Drain::start(Answer::BusyUntil {
14451            until: Instant::now() + Duration::from_millis(600),
14452            latency: Duration::ZERO,
14453        });
14454        let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14455        assert!(drained, "the gauge fell at 600ms: {trace}");
14456        assert!(
14457            elapsed < Duration::from_secs(3),
14458            "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14459        );
14460        let probes = drain.probes();
14461        assert!(
14462            probes.len() >= 2 && trace.probes as usize == probes.len(),
14463            "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14464        );
14465    }
14466
14467    #[tokio::test(start_paused = true)]
14468    async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14469        let started = Instant::now();
14470        let drain = Drain::start(Answer::BusyUntil {
14471            until: started + Duration::from_secs(3600),
14472            latency: Duration::ZERO,
14473        });
14474        let (drained, elapsed) = drain.run(CEILING).await;
14475        assert!(!drained, "a module busy throughout must not drain");
14476        assert!(
14477            elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14478            "a busy module is held until the ceiling and no longer: {elapsed:?}"
14479        );
14480        // Re-probing is paced by the interval, not a busy loop: at most one
14481        // probe per interval across the ceiling, plus the first at time zero.
14482        let probes = drain.probes();
14483        let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14484        assert!(
14485            probes.len() as u128 <= max_probes,
14486            "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14487            probes.len()
14488        );
14489        for pair in probes.windows(2) {
14490            assert!(
14491                pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14492                "consecutive probes {:?} apart",
14493                pair[1] - pair[0]
14494            );
14495        }
14496    }
14497
14498    #[tokio::test(start_paused = true)]
14499    async fn a_slow_probe_is_never_overlapped_by_the_next() {
14500        // Each answer takes longer than the re-probe interval; the next probe
14501        // must wait for the previous answer and then the interval.
14502        let latency = Duration::from_secs(1);
14503        let drain = Drain::start(Answer::BusyUntil {
14504            until: Instant::now() + Duration::from_secs(3600),
14505            latency,
14506        });
14507        let (drained, _) = drain.run(Duration::from_secs(5)).await;
14508        assert!(!drained);
14509        let probes = drain.probes();
14510        assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14511        for pair in probes.windows(2) {
14512            assert!(
14513                pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14514                "a probe started {:?} after the previous one, before it could finish",
14515                pair[1] - pair[0]
14516            );
14517        }
14518    }
14519
14520    #[tokio::test(start_paused = true)]
14521    async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14522        let drain = Drain::start(Answer::Silent);
14523        let ceiling = Duration::from_secs(2);
14524        let (drained, elapsed) = drain.run(ceiling).await;
14525        assert!(!drained, "an unanswered probe counts as busy");
14526        assert!(
14527            elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14528            "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14529        );
14530        assert_eq!(
14531            drain.probes().len(),
14532            1,
14533            "the unanswered probe holds the only slot until the ceiling"
14534        );
14535    }
14536
14537    /// Every event logged while it is installed, as `message` plus fields.
14538    #[derive(Clone, Default)]
14539    struct LogCapture {
14540        events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14541    }
14542
14543    impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14544        fn on_event(
14545            &self,
14546            event: &tracing::Event<'_>,
14547            _context: tracing_subscriber::layer::Context<'_, S>,
14548        ) {
14549            struct Fields(BTreeMap<String, String>);
14550            impl tracing::field::Visit for Fields {
14551                fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14552                    self.0
14553                        .insert(field.name().to_string(), format!("{value:?}"));
14554                }
14555            }
14556            let mut fields = Fields(BTreeMap::new());
14557            event.record(&mut fields);
14558            self.events.lock().unwrap().push(fields.0);
14559        }
14560    }
14561
14562    impl LogCapture {
14563        fn install(&self) -> tracing::subscriber::DefaultGuard {
14564            use tracing_subscriber::layer::SubscriberExt;
14565            tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14566        }
14567
14568        fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14569            self.events
14570                .lock()
14571                .unwrap()
14572                .iter()
14573                .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14574                .cloned()
14575                .collect()
14576        }
14577    }
14578
14579    /// A module that declares a busy gauge but leaves it out of its health
14580    /// answers holds a drain to its ceiling. The timeout line must say so by
14581    /// name, and the drain must warn about the missing gauge once, not on
14582    /// every probe.
14583    #[tokio::test(start_paused = true)]
14584    async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14585        let capture = LogCapture::default();
14586        let _guard = capture.install();
14587        let drain = Drain::start(Answer::OmitsGauge);
14588        let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14589        assert!(!drained, "a gauge never reported cannot read quiet");
14590        assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14591
14592        let missing = capture.with_message("lacks a declared busy gauge");
14593        assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14594        assert_eq!(
14595            missing[0].get("missing_gauges").map(String::as_str),
14596            Some(GAUGE)
14597        );
14598
14599        warn_drain_timed_out(
14600            MODULE_ID,
14601            Duration::from_secs(2),
14602            RouteCloseReason::Restart,
14603            &Default::default(),
14604            &trace,
14605        );
14606        let timeouts = capture.with_message("route drain timed out");
14607        assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14608        let gauges = &timeouts[0]["busy_gauges"];
14609        assert!(
14610            gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14611            "the timeout line must name the gauge and why it read busy: {gauges}"
14612        );
14613        assert!(
14614            gauges.ends_with("; no value ever read"),
14615            "no value was ever read for the gauge: {gauges}"
14616        );
14617        assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14618    }
14619
14620    /// The trace names a busy value and when it was read. For a probe that
14621    /// never got an answer, it names the probe's error kind and says that no
14622    /// value was ever read.
14623    #[tokio::test(start_paused = true)]
14624    async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14625        let busy = Drain::start(Answer::BusyUntil {
14626            until: Instant::now() + Duration::from_secs(3600),
14627            latency: Duration::ZERO,
14628        });
14629        let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14630        let rendered = trace.to_string();
14631        assert!(
14632            rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14633                && rendered.contains("; last value non_zero(1) @"),
14634            "{rendered}"
14635        );
14636
14637        let silent = Drain::start(Answer::Silent);
14638        let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14639        // The probe is capped at the 1 s ceiling, and its reading is stamped
14640        // when it gave up.
14641        assert_eq!(
14642            trace.to_string(),
14643            format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14644        );
14645        assert_eq!(trace.probes, 1);
14646    }
14647
14648    /// A gauge value that is not a count reads busy and is shown as sent.
14649    #[test]
14650    fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14651        let gauges = [GAUGE.to_string()];
14652        let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14653        assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14654        assert_eq!(
14655            busy_gauge_observation(&verdicts),
14656            BusyGaugeObservation::Busy
14657        );
14658        let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14659        assert_eq!(
14660            busy_gauge_observation(&zero),
14661            BusyGaugeObservation::Quiescent
14662        );
14663        let none = gauge_verdicts(None, &gauges);
14664        assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14665    }
14666}
14667
14668#[cfg(test)]
14669mod health_evidence_tests {
14670    use super::{HealthProbeError, HealthProbeEvidence};
14671    use std::collections::HashSet;
14672
14673    /// The evidential asymmetry, asserted rather than described.
14674    ///
14675    /// Exactly ONE observation is proof a module cannot serve, and the one that
14676    /// fires under CPU starvation is not it. Before the split, all fifteen
14677    /// construction sites collapsed into a single String, so a timeout carried the
14678    /// same weight as a dead lane -- which is how a healthy module was restarted
14679    /// three times in one day.
14680    #[test]
14681    fn only_a_dead_lane_is_proof_of_death() {
14682        assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14683        // Three non-proof classes, each for a different reason: silence is
14684        // consistent with health, a bad answer proves the module ALIVE, and a
14685        // daemon-side fault never reached the module at all.
14686        assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14687        assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14688        assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14689    }
14690
14691    /// Labels must be distinct, or the operator-facing distinction is cosmetic.
14692    ///
14693    /// A shared label renders two different observations identically in the line an
14694    /// operator reads after an unexplained restart -- the exact confusion this
14695    /// change removes.
14696    #[test]
14697    fn every_evidence_class_has_a_distinct_label() {
14698        let labels = [
14699            HealthProbeError::lane_dead("").label(),
14700            HealthProbeError::no_answer("").label(),
14701            HealthProbeError::bad_answer("").label(),
14702            HealthProbeError::misconfigured("").label(),
14703        ];
14704        let unique: HashSet<_> = labels.iter().collect();
14705        assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14706    }
14707
14708    /// The class is additional information, not a replacement.
14709    ///
14710    /// An operator needs both "this was silence" and the specific text saying how
14711    /// long we waited; a classification that swallowed the message would trade one
14712    /// missing distinction for another.
14713    #[test]
14714    fn classification_preserves_the_original_message() {
14715        let err = HealthProbeError::no_answer("module did not answer within 5s");
14716        assert_eq!(err.to_string(), "module did not answer within 5s");
14717        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14718    }
14719}
14720
14721#[cfg(test)]
14722mod health_tombstone_tests {
14723    use std::{path::PathBuf, sync::Arc, time::Duration};
14724
14725    use subc_protocol::{
14726        manifest::Concurrency,
14727        session::{HealthStatus, ModuleControlResponse},
14728    };
14729    use tokio::sync::mpsc;
14730
14731    use super::{
14732        probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14733        ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14734    };
14735    use crate::{
14736        control::ControlHandler,
14737        forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14738        registry::{ConnectionId, Registry},
14739        router::FrameSink,
14740    };
14741
14742    struct ProbeHarness {
14743        spec: ModuleSpec,
14744        runtime: SupervisorRuntimeConfig,
14745        forwarding: Arc<ForwardingTable>,
14746        module_connection: ConnectionId,
14747        module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14748        handler: ControlHandler,
14749        module: super::SupervisedModule,
14750    }
14751
14752    fn probe_harness() -> ProbeHarness {
14753        let registry = Arc::new(Registry::default());
14754        let forwarding = Arc::new(ForwardingTable::default());
14755        let supervisor_handle = super::SupervisorHandle::new();
14756        let health = HealthConfig {
14757            http: None,
14758            cadence: Duration::from_secs(30),
14759            deadline: Duration::from_secs(5),
14760            failure_threshold: 3,
14761            on_degraded: HealthAction::Report,
14762            on_failing: HealthAction::Report,
14763            critical: false,
14764        };
14765        let supervisor = Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
14766            .with_forwarding(Arc::clone(&forwarding))
14767            .with_handle(supervisor_handle.clone())
14768            .with_health_config(health);
14769        let spec = ModuleSpec {
14770            module_id: "late-health-module".to_string(),
14771            program: PathBuf::from("disabled-module"),
14772            args: Vec::new(),
14773            env: Vec::new(),
14774            reserved: false,
14775            reserved_prefixes: Vec::new(),
14776            protocol: ModuleProtocol::Subc,
14777            overlap: Default::default(),
14778        };
14779        let module = supervisor
14780            .supervise_configured(spec.clone(), false)
14781            .unwrap();
14782        let runtime = supervisor.runtime_config();
14783        let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14784            .with_supervisor(supervisor_handle);
14785        let module_connection = ConnectionId::new(700);
14786        let (module_tx, module_rx) = mpsc::channel(8);
14787        forwarding
14788            .register_module_connection(
14789                module_connection,
14790                spec.module_id.clone(),
14791                subc_protocol::PROTOCOL_VERSION,
14792                Concurrency::ModuleManaged,
14793                FrameSink::new(module_tx),
14794            )
14795            .unwrap();
14796
14797        ProbeHarness {
14798            spec,
14799            runtime,
14800            forwarding,
14801            module_connection,
14802            module_rx,
14803            handler,
14804            module,
14805        }
14806    }
14807
14808    async fn finish_after(
14809        harness: &mut ProbeHarness,
14810        stall: Duration,
14811    ) -> ModuleControlRpcCompletion {
14812        assert!(stall > harness.runtime.health.deadline);
14813        let deadline = harness.runtime.health.deadline;
14814        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14815        let answer = async {
14816            let frame = harness.module_rx.recv().await.expect("health.check frame");
14817            tokio::time::advance(deadline).await;
14818            tokio::task::yield_now().await;
14819            tokio::time::advance(stall - deadline).await;
14820            harness
14821                .forwarding
14822                .complete_module_control_rpc(
14823                    harness.module_connection,
14824                    frame.header.corr,
14825                    Some("health.check"),
14826                    ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14827                        status: HealthStatus::Ok,
14828                        detail: None,
14829                        metrics: None,
14830                    }),
14831                )
14832                .unwrap()
14833        };
14834        let (probe_result, completion) = tokio::join!(probe, answer);
14835        let err = probe_result.expect_err("probe must miss its deadline");
14836        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14837        completion
14838    }
14839
14840    async fn time_out_without_answer(harness: &mut ProbeHarness) {
14841        let deadline = harness.runtime.health.deadline;
14842        let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14843        let exhaust_deadline = async {
14844            let _frame = harness.module_rx.recv().await.expect("health.check frame");
14845            tokio::time::advance(deadline).await;
14846            tokio::task::yield_now().await;
14847        };
14848        let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14849        let err = probe_result.expect_err("probe must miss its deadline");
14850        assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14851    }
14852
14853    async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14854        let registry = Arc::clone(&harness.module.inner.registry);
14855        let snapshot = Arc::clone(&harness.module.inner.snapshot);
14856        let process_liveness = super::SupervisorProcessLiveness::default();
14857        let mut child = None;
14858        let cycle = super::run_health_probe_cycle(
14859            &harness.spec,
14860            &harness.runtime,
14861            &registry,
14862            &process_liveness,
14863            &snapshot,
14864            &mut child,
14865        );
14866        let peer = async {
14867            let frame = harness.module_rx.recv().await.expect("health.check frame");
14868            if answer {
14869                harness
14870                    .forwarding
14871                    .complete_module_control_rpc(
14872                        harness.module_connection,
14873                        frame.header.corr,
14874                        Some("health.check"),
14875                        ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14876                            status: HealthStatus::Ok,
14877                            detail: None,
14878                            metrics: Some(serde_json::json!({"ready": true})),
14879                        }),
14880                    )
14881                    .unwrap();
14882            } else {
14883                tokio::time::advance(harness.runtime.health.deadline).await;
14884                tokio::task::yield_now().await;
14885            }
14886        };
14887        tokio::join!(cycle, peer);
14888    }
14889
14890    #[tokio::test(start_paused = true)]
14891    async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14892        let mut harness = probe_harness();
14893        // Drive the probe cycle directly with an in-memory wire peer. Stop the
14894        // disabled module's monitor so only this test owns lifecycle transitions;
14895        // no OS process is launched, and a restart is observed at scheduling.
14896        let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14897        monitor.abort();
14898        let _ = monitor.await;
14899        super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14900            state.enabled = true;
14901            state.state = super::ModuleState::Running;
14902            state.process_alive = true;
14903        })
14904        .unwrap();
14905
14906        run_probe_cycle(&mut harness, true).await;
14907        assert_eq!(
14908            harness.module.status().unwrap().health.status,
14909            super::SupervisorHealthStatus::Ok
14910        );
14911
14912        for failures in 1..harness.runtime.health.failure_threshold {
14913            run_probe_cycle(&mut harness, false).await;
14914            let status = harness.module.status().unwrap();
14915            assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14916            assert_eq!(status.health.consecutive_failures, failures);
14917            assert!(status.health.last_probe_ms.is_some());
14918            assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14919            assert!(status.health.metrics.is_none());
14920            assert_eq!(status.state, super::ModuleState::Running);
14921            assert!(status.process_alive);
14922            assert_eq!(status.restart_count, 0);
14923            assert_eq!(status.lifetime_restarts, 0);
14924            assert!(status.health.last_action.is_none());
14925            assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14926        }
14927
14928        run_probe_cycle(&mut harness, true).await;
14929        let recovered = harness.module.status().unwrap();
14930        assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14931        assert_eq!(recovered.health.consecutive_failures, 0);
14932        assert!(recovered.health.detail.is_none());
14933        assert_eq!(
14934            recovered.health.metrics,
14935            Some(serde_json::json!({"ready": true}))
14936        );
14937        assert_eq!(recovered.lifetime_restarts, 0);
14938
14939        for failures in 1..=harness.runtime.health.failure_threshold {
14940            run_probe_cycle(&mut harness, false).await;
14941            let status = harness.module.status().unwrap();
14942            assert_eq!(status.health.consecutive_failures, failures);
14943            if failures < harness.runtime.health.failure_threshold {
14944                assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14945                assert_eq!(status.state, super::ModuleState::Running);
14946                assert_eq!(status.lifetime_restarts, 0);
14947                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14948            } else {
14949                assert_eq!(
14950                    status.health.status,
14951                    super::SupervisorHealthStatus::Unresponsive
14952                );
14953                assert_eq!(status.state, super::ModuleState::Restarting);
14954                assert_eq!(status.restart_count, 1);
14955                assert_eq!(status.lifetime_restarts, 1);
14956                assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14957                assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14958            }
14959        }
14960    }
14961
14962    #[tokio::test(start_paused = true)]
14963    async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14964        let mut harness = probe_harness();
14965
14966        let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14967        let first_latency = match &first {
14968            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14969            other => panic!("late answer was not retained: {other:?}"),
14970        };
14971        assert!(harness.handler.observe_module_control_completion(first));
14972
14973        let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14974        let second_latency = match &second {
14975            ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14976            other => panic!("late answer was not retained: {other:?}"),
14977        };
14978        assert!(harness.handler.observe_module_control_completion(second));
14979
14980        assert_eq!(first_latency, Duration::from_secs(8));
14981        assert_eq!(
14982            second_latency - first_latency,
14983            Duration::from_secs(3),
14984            "latency must grow linearly with the additional stall"
14985        );
14986        let health = harness.module.status().unwrap().health;
14987        assert_eq!(health.late_answer_count, 2);
14988        assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14989    }
14990
14991    /// A module that answers every probe late must never march to the kill
14992    /// threshold: the late answer proves it is alive, so it must clear the miss
14993    /// streak the timeout recorded. Without the reset, a CPU-starved module
14994    /// that serves every probe seconds past the deadline accumulates
14995    /// `consecutive_failures` to the threshold and is killed — the exact
14996    /// sequence from the 2026-08-14 aft disable, where the daemon logged
14997    /// "proves the module is alive" five times while counting five misses.
14998    #[tokio::test(start_paused = true)]
14999    async fn late_answer_clears_the_consecutive_failure_streak() {
15000        let mut harness = probe_harness();
15001
15002        // Timeout recorded first: the probe path saw no answer in time.
15003        time_out_without_answer(&mut harness).await;
15004        harness
15005            .module
15006            .record_health_probe_failure_for_test("[no-answer] test miss")
15007            .unwrap();
15008        assert_eq!(
15009            harness.module.status().unwrap().health.consecutive_failures,
15010            1,
15011            "precondition: the miss must be on the streak before the late answer"
15012        );
15013
15014        // The stalled reply then lands: proof of life.
15015        let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15016        assert!(matches!(
15017            late,
15018            ModuleControlRpcCompletion::LateHealthAnswer { .. }
15019        ));
15020        assert!(harness.handler.observe_module_control_completion(late));
15021
15022        let health = harness.module.status().unwrap().health;
15023        assert_eq!(
15024            health.consecutive_failures, 0,
15025            "a late answer is an answer: the streak must reset"
15026        );
15027        assert_eq!(health.late_answer_count, 1);
15028    }
15029
15030    #[tokio::test(start_paused = true)]
15031    async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15032        let mut harness = probe_harness();
15033
15034        for _ in 0..20 {
15035            time_out_without_answer(&mut harness).await;
15036            assert_eq!(
15037                harness.forwarding.health_probe_tombstone_count().unwrap(),
15038                1
15039            );
15040        }
15041    }
15042}
15043
15044#[cfg(test)]
15045mod child_env_tests {
15046    #[cfg(windows)]
15047    use super::{
15048        clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15049    };
15050    #[cfg(windows)]
15051    use std::sync::{Arc, OnceLock};
15052    #[cfg(windows)]
15053    #[test]
15054    fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15055        let spec = spec(Vec::new());
15056        let mut command = Command::new("unused");
15057        let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15058            .unwrap()
15059            .unwrap();
15060        let name = command
15061            .as_std()
15062            .get_envs()
15063            .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15064            .and_then(|(_, value)| value)
15065            .unwrap();
15066        assert_eq!(name, OsStr::new(handoff.name()));
15067        assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15068        assert!(command.as_std().get_envs().any(|(key, value)| key
15069            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15070            && value == Some(OsStr::new("env"))));
15071        assert!(command
15072            .as_std()
15073            .get_envs()
15074            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15075    }
15076
15077    #[cfg(windows)]
15078    #[test]
15079    fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15080        let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15081        snapshot.process_alive = true;
15082        snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15083        assert_eq!(
15084            observed_launch_nonce_source(&snapshot).as_deref(),
15085            Some("env")
15086        );
15087        let consumed = Arc::new(OnceLock::new());
15088        snapshot.pipe_consumed = Some(consumed.clone());
15089        assert_eq!(
15090            observed_launch_nonce_source(&snapshot).as_deref(),
15091            Some("env")
15092        );
15093        consumed.set(()).unwrap();
15094        assert_eq!(
15095            observed_launch_nonce_source(&snapshot).as_deref(),
15096            Some("pipe")
15097        );
15098        snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15099        assert_eq!(
15100            observed_launch_nonce_source(&snapshot).as_deref(),
15101            Some("env")
15102        );
15103        clear_current_process_facts(&mut snapshot);
15104        assert_eq!(observed_launch_nonce_source(&snapshot), None);
15105    }
15106    use super::{
15107        apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15108        ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15109        SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15110    };
15111    use std::{ffi::OsStr, path::PathBuf};
15112    use tokio::process::Command;
15113
15114    fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15115        ModuleSpec {
15116            module_id: "env-plan".to_string(),
15117            program: PathBuf::from("/nonexistent"),
15118            args: Vec::new(),
15119            env,
15120            reserved: false,
15121            reserved_prefixes: Vec::new(),
15122            protocol: ModuleProtocol::Subc,
15123            overlap: Default::default(),
15124        }
15125    }
15126
15127    #[test]
15128    fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15129        let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15130        let digest = LaunchSpecDigest::from_spec(&base);
15131        let mut reordered = base.clone();
15132        reordered.env.reverse();
15133        reordered.module_id = "different-id".into();
15134        reordered.reserved = true;
15135        reordered.overlap = Default::default();
15136        assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15137        let mut delimiter_collision = base.clone();
15138        delimiter_collision.env[0] = ("A=B".into(), "C".into());
15139        assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15140        let mut changed = base.clone();
15141        changed.protocol = ModuleProtocol::None;
15142        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15143        changed = base.clone();
15144        changed.args.push("--flag".into());
15145        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15146        changed = base.clone();
15147        changed.program = PathBuf::from("/other");
15148        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15149        changed = base.clone();
15150        changed.env[1].1 = "other".into();
15151        assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15152        assert!(!format!("{digest:?}").contains("B=C"));
15153        assert!(!format!("{digest:?}").contains("[u8"));
15154    }
15155
15156    #[cfg(unix)]
15157    #[test]
15158    fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15159        use std::os::unix::ffi::OsStringExt;
15160        let mut left = spec(Vec::new());
15161        left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15162        let mut right = left.clone();
15163        right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15164        assert_eq!(
15165            left.program.to_string_lossy(),
15166            right.program.to_string_lossy()
15167        );
15168        assert_ne!(
15169            LaunchSpecDigest::from_spec(&left),
15170            LaunchSpecDigest::from_spec(&right)
15171        );
15172    }
15173
15174    /// Ambient `CK_LOG` is REMOVED for an unconfigured module, and a configured
15175    /// one still gets its own.
15176    ///
15177    /// This is the narrow goal `env_clear()` was reached for, and the reason the
15178    /// fix is `env_remove` rather than deleting the line: an operator's ambient
15179    /// filter silently becoming an unconfigured module's log level is a real
15180    /// defect, just a much smaller one than clearing the environment.
15181    ///
15182    /// Asserted on the command plan rather than a spawned child because proving
15183    /// the ABSENCE of an inherited variable needs the parent's environment
15184    /// mutated, and `forbid(unsafe_code)` refuses that. `get_envs()` reports a
15185    /// removal as `(key, None)`, which is exactly the distinction wanted: not
15186    /// "absent because nobody set it" but "explicitly unset for the child".
15187    #[test]
15188    fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15189        let mut command = Command::new("/nonexistent");
15190        apply_child_env(&mut command, &spec(Vec::new()));
15191        let removed = command
15192            .as_std()
15193            .get_envs()
15194            .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15195        assert!(
15196            removed,
15197            "ambient CK_LOG must be explicitly removed for an unconfigured module"
15198        );
15199
15200        let mut configured = Command::new("/nonexistent");
15201        apply_child_env(
15202            &mut configured,
15203            &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15204        );
15205        let effective = configured
15206            .as_std()
15207            .get_envs()
15208            .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15209            .last()
15210            .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15211        assert_eq!(
15212            effective,
15213            Some(Some("debug".to_string())),
15214            "a module's configured CK_LOG must survive the ambient removal"
15215        );
15216    }
15217
15218    /// A `protocol: "none"` spawn carries NO `--subc` argument and NO launch
15219    /// nonce; a subc-wire spawn carries both. Asserted on the command plan for
15220    /// the same reason as the CK_LOG test above.
15221    ///
15222    /// The argument is the load-bearing half: a stock binary exits on an
15223    /// unknown flag before it listens, so with `--subc` appended the mode
15224    /// could not supervise the one process it exists for. Found by the first
15225    /// conformance run (nats-server: `flag provided but not defined: -subc`).
15226    #[test]
15227    fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15228        let connection_file = std::path::Path::new("/run/subc-connection.json");
15229        let handle = SupervisorHandle::new();
15230
15231        let mut none_spec = spec(Vec::new());
15232        none_spec.protocol = ModuleProtocol::None;
15233        let mut none = Command::new("/nonexistent");
15234        let none_handoff =
15235            apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15236                .expect("protocol-none spawn args apply");
15237        assert!(
15238            none_handoff.is_none(),
15239            "protocol:none spawn must not receive a nonce descriptor"
15240        );
15241        #[cfg(windows)]
15242        assert!(!none.as_std().get_envs().any(|(key, value)| key
15243            == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15244            && value.is_some()));
15245        assert!(
15246            !none.as_std().get_envs().any(|(key, value)| key
15247                == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15248                && value.is_some()),
15249            "protocol:none spawn must not name a nonce descriptor"
15250        );
15251        let none_args: Vec<String> = none
15252            .as_std()
15253            .get_args()
15254            .map(|a| a.to_string_lossy().into_owned())
15255            .collect();
15256        assert!(
15257            !none_args.iter().any(|a| a == SUBC_ARG),
15258            "protocol:none argv must not carry --subc; got {none_args:?}"
15259        );
15260        let none_has_nonce = none
15261            .as_std()
15262            .get_envs()
15263            .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15264        assert!(
15265            !none_has_nonce,
15266            "protocol:none spawn must not receive a launch nonce"
15267        );
15268        let none_has_module_id = none
15269            .as_std()
15270            .get_envs()
15271            .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15272        assert!(
15273            none_has_module_id,
15274            "SUBC_MODULE_ID is inert and stays on every path"
15275        );
15276        assert!(
15277            handle.spawn_nonce(&none_spec.module_id).is_none(),
15278            "no nonce record for a process that will never present one"
15279        );
15280
15281        // Control: the subc-wire path is unchanged by the branch above.
15282        let wire_spec = spec(Vec::new());
15283        let mut wire = Command::new("/nonexistent");
15284        let wire_handoff =
15285            apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15286                .expect("subc-wire spawn args apply");
15287        let wire_fd_env = wire
15288            .as_std()
15289            .get_envs()
15290            .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15291            .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15292        #[cfg(unix)]
15293        assert_eq!(
15294            wire_fd_env,
15295            Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15296            "a subc-wire spawn names the pipe it will receive at descriptor 3"
15297        );
15298        #[cfg(windows)]
15299        assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15300        #[cfg(not(any(unix, windows)))]
15301        assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15302        let wire_args: Vec<String> = wire
15303            .as_std()
15304            .get_args()
15305            .map(|a| a.to_string_lossy().into_owned())
15306            .collect();
15307        assert_eq!(
15308            wire_args,
15309            vec![
15310                SUBC_ARG.to_string(),
15311                connection_file.to_string_lossy().into_owned()
15312            ],
15313            "a subc-wire spawn still carries --subc <path>"
15314        );
15315        assert_eq!(
15316            wire.as_std()
15317                .get_envs()
15318                .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15319            !cfg!(unix),
15320            "only Windows supplies the environment nonce"
15321        );
15322        assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15323    }
15324
15325    /// A plain spawn EXPLICITLY REMOVES the spawn role, even when the module's
15326    /// spec tries to set it; only a swap candidate carries it.
15327    ///
15328    /// "Set it only on candidates" is not enough, because spawn applies the
15329    /// spec's env verbatim and the daemon's own environment is inherited: either
15330    /// could hand a plain restart the swap role, and a module reading it would
15331    /// warm on its long swap budget while callers wait. Asserted as an explicit
15332    /// removal (`(key, None)`), not mere absence, for the reason the `CK_LOG`
15333    /// test above gives.
15334    #[test]
15335    fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15336        let role = |command: &Command| {
15337            command
15338                .as_std()
15339                .get_envs()
15340                .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15341                .last()
15342                .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15343        };
15344        let forged = spec(vec![(
15345            SUBC_SPAWN_ROLE_ENV.to_string(),
15346            SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15347        )]);
15348
15349        let mut plain = Command::new("/nonexistent");
15350        apply_child_env(&mut plain, &forged);
15351        apply_spawn_role(&mut plain, SpawnRole::Plain);
15352        assert_eq!(
15353            role(&plain),
15354            Some(None),
15355            "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15356        );
15357
15358        let mut candidate = Command::new("/nonexistent");
15359        apply_child_env(&mut candidate, &spec(Vec::new()));
15360        apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15361        assert_eq!(
15362            role(&candidate),
15363            Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15364        );
15365    }
15366
15367    /// Daemon-private capture retention keys never reach the child.
15368    ///
15369    /// cortexkit-log exposes retention as a Rust struct with no environment
15370    /// names, so these entries are supervisor metadata. Passing them through
15371    /// would invent a public child-process contract by accident.
15372    #[test]
15373    fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15374        let mut command = Command::new("/nonexistent");
15375        apply_child_env(
15376            &mut command,
15377            &spec(vec![
15378                (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15379                ("KEPT".to_string(), "yes".to_string()),
15380            ]),
15381        );
15382        let keys: Vec<String> = command
15383            .as_std()
15384            .get_envs()
15385            .filter(|(_, value)| value.is_some())
15386            .map(|(key, _)| key.to_string_lossy().into_owned())
15387            .collect();
15388        assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15389        assert!(
15390            !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15391            "daemon-private capture key leaked to the child: {keys:?}"
15392        );
15393    }
15394}
15395
15396#[cfg(test)]
15397mod jitter_tests {
15398    use super::jittered_health_delay;
15399    use std::{collections::HashSet, time::Duration};
15400
15401    /// Module ids drawn from a real fleet, so the dispersal claim is about names
15402    /// that actually occur rather than invented ones.
15403    ///
15404    /// This is a SAMPLE, not a registry: the property under test is that distinct
15405    /// ids disperse, which holds for any set of distinct strings. Several entries
15406    /// are already historical (modules get renamed), and that costs nothing here --
15407    /// but it means a reader must not mistake this for the live module set, and a
15408    /// rename sweep will match it without there being anything to change.
15409    const FLEET: [&str; 14] = [
15410        "aft",
15411        "alfonso-core",
15412        "magic-context",
15413        "broca",
15414        "thalamus",
15415        "quota",
15416        "engram",
15417        "plexus",
15418        "cerebellum",
15419        "astrocyte",
15420        "synapse",
15421        "subc-mcp",
15422        "cortexkit-credentials",
15423        "subc-federation",
15424    ];
15425
15426    /// Probes must not converge after a fleet-wide restart.
15427    ///
15428    /// This is the property the jitter exists for: every module reconnects at
15429    /// once, and without dispersal all fourteen would then probe on the same
15430    /// tick forever. Nothing failed visibly when this went untested -- a
15431    /// convergent fleet still probes correctly, just in a burst, so the symptom
15432    /// is a periodic load spike that looks like whatever else is running.
15433    #[test]
15434    fn probe_delays_disperse_across_the_fleet() {
15435        let cadence = Duration::from_secs(30);
15436        let delays: HashSet<Duration> = FLEET
15437            .iter()
15438            .map(|id| jittered_health_delay(id, 0, cadence))
15439            .collect();
15440        assert_eq!(
15441            delays.len(),
15442            FLEET.len(),
15443            "every supervised module must land on its own probe offset"
15444        );
15445    }
15446
15447    /// The offset may only ever DELAY a probe, never bring it forward.
15448    ///
15449    /// A delay below the cadence would probe a module more often than
15450    /// configured, which is the opposite of what an operator asked for and
15451    /// would tighten the failure budget without anyone changing it.
15452    #[test]
15453    fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15454        let cadence = Duration::from_secs(30);
15455        let span = cadence / 10;
15456        for id in FLEET {
15457            for probe_index in 0..8 {
15458                let delay = jittered_health_delay(id, probe_index, cadence);
15459                assert!(
15460                    delay >= cadence,
15461                    "{id}#{probe_index}: jitter must not shorten the cadence"
15462                );
15463                assert!(
15464                    delay < cadence + span,
15465                    "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15466                );
15467            }
15468        }
15469    }
15470
15471    /// A module keeps its offset across daemon restarts.
15472    ///
15473    /// The delay is derived rather than randomised precisely so a restart does
15474    /// not re-roll every module into a fresh chance of collision. A random
15475    /// source would satisfy the dispersal test above and quietly lose this.
15476    #[test]
15477    fn a_module_offset_is_stable_across_restarts() {
15478        let cadence = Duration::from_secs(30);
15479        for id in FLEET {
15480            assert_eq!(
15481                jittered_health_delay(id, 0, cadence),
15482                jittered_health_delay(id, 0, cadence),
15483                "{id}: the same module and probe index must produce the same offset"
15484            );
15485        }
15486    }
15487
15488    /// A zero cadence disables probing rather than producing a busy loop.
15489    #[test]
15490    fn zero_cadence_yields_zero_delay() {
15491        assert_eq!(
15492            jittered_health_delay("aft", 0, Duration::ZERO),
15493            Duration::ZERO
15494        );
15495    }
15496}
15497
15498#[cfg(all(test, target_os = "linux"))]
15499mod cgroup_placement_tests {
15500    use super::{
15501        apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15502        SuperviseError, SupervisedChild,
15503    };
15504    use crate::stderr_tail::{StderrRing, StderrTailConfig};
15505    use cortexkit_test_support::ScratchDir;
15506    use std::{
15507        fs, io,
15508        path::{Path, PathBuf},
15509        sync::{Arc, Mutex},
15510    };
15511    use tokio::process::Command;
15512
15513    #[tokio::test]
15514    async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15515        use super::*;
15516        let dir = ScratchDir::new("unique-spawn-cgroups");
15517        let root = PathBuf::from(format!(
15518            "/sys/fs/cgroup/subc-unique-test-{}-{}",
15519            std::process::id(),
15520            unix_ms_now()
15521        ));
15522        if let Err(error) = fs::create_dir(&root) {
15523            assert!(
15524                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15525                "required cgroup test cannot execute: {error}"
15526            );
15527            eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15528            return;
15529        }
15530        let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15531        let group_count = || {
15532            fs::read_dir(root.join("subc-modules"))
15533                .unwrap()
15534                .map(|entry| entry.unwrap().file_type().unwrap())
15535                .filter(|kind| kind.is_dir())
15536                .count()
15537        };
15538        let supervisor =
15539            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15540                .with_cgroup_placement(Some(placement.clone()));
15541        let runtime = supervisor.runtime_config();
15542        let mut spec = ModuleSpec {
15543            module_id: "unique-spawn".into(),
15544            program: PathBuf::from("/bin/sleep"),
15545            args: vec!["60".into()],
15546            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15547                .into_iter()
15548                .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15549                .collect(),
15550            reserved: false,
15551            reserved_prefixes: vec![],
15552            protocol: ModuleProtocol::None,
15553            overlap: Default::default(),
15554        };
15555        let spawn = |spec: &ModuleSpec| {
15556            spawn_child(
15557                spec,
15558                None,
15559                None,
15560                &runtime.stderr_ring,
15561                None,
15562                &runtime.child_roster,
15563                Some(&placement),
15564            )
15565            .unwrap()
15566        };
15567        let mut live = spawn(&spec);
15568        for _ in 0..3 {
15569            // A new process can enter the old slot while retirement is pending.
15570            let next = spawn(&spec);
15571            assert_ne!(live.module_id, next.module_id);
15572            live.start_kill().unwrap();
15573            live.wait().await.unwrap();
15574            live = next;
15575            assert!(
15576                live.child.try_wait().unwrap().is_none(),
15577                "retiring the old slot must not kill the replacement"
15578            );
15579            assert_eq!(
15580                group_count(),
15581                1,
15582                "only the live spawn's cgroup should remain"
15583            );
15584        }
15585        supervisor.begin_daemon_shutdown();
15586        let reap = tokio::spawn(async move {
15587            live.wait().await.unwrap();
15588        });
15589        supervisor
15590            .end_children_for_daemon_shutdown(false, std::future::pending())
15591            .await;
15592        reap.await.unwrap();
15593        assert_eq!(group_count(), 0);
15594        // A normal exit uses the same tree-cleanup path as a killed spawn.
15595        spec.program = PathBuf::from("/bin/true");
15596        spec.args.clear();
15597        let fresh_roster = ChildRoster::default();
15598        let mut short = spawn_child(
15599            &spec,
15600            None,
15601            None,
15602            &runtime.stderr_ring,
15603            None,
15604            &fresh_roster,
15605            Some(&placement),
15606        )
15607        .unwrap();
15608        short.wait().await.unwrap();
15609        assert_eq!(group_count(), 0);
15610        spec.module_id = "_".repeat(255);
15611        let mut long_id = spawn_child(
15612            &spec,
15613            None,
15614            None,
15615            &runtime.stderr_ring,
15616            None,
15617            &fresh_roster,
15618            Some(&placement),
15619        )
15620        .unwrap();
15621        long_id.wait().await.unwrap();
15622        assert_eq!(
15623            group_count(),
15624            0,
15625            "valid long module IDs must not exceed cgroup NAME_MAX"
15626        );
15627        fs::remove_dir(root.join("subc-modules")).unwrap();
15628        fs::remove_dir(root).unwrap();
15629        eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15630    }
15631
15632    #[test]
15633    fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15634        let path = Path::new("/definitely-missing-subc-cgroup");
15635        let mut command = Command::new("true");
15636        let error = apply_cgroup_placement(
15637            &mut command,
15638            &ModuleSpec {
15639                module_id: "broken-cgroup".to_string(),
15640                program: PathBuf::from("true"),
15641                args: Vec::new(),
15642                env: Vec::new(),
15643                reserved: false,
15644                reserved_prefixes: Vec::new(),
15645                protocol: ModuleProtocol::Subc,
15646                overlap: Default::default(),
15647            },
15648            path,
15649        )
15650        .expect_err("a parent cgroup open failure must reject the supervised spawn");
15651        let reason = error.to_string();
15652
15653        assert!(
15654            matches!(error, SuperviseError::Cgroup { .. }),
15655            "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15656        );
15657        assert!(
15658            reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15659            "parent cgroup open failure must name cgroup.procs: {reason}"
15660        );
15661    }
15662
15663    #[tokio::test]
15664    async fn reaping_a_child_removes_its_empty_module_cgroup() {
15665        let root = ScratchDir::new("supervisor-reap-cgroup");
15666        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15667        let placement = subc_cgroup::prepare_at(&root)
15668            .expect("prepare scratch cgroup root")
15669            .expect("scratch root has a cgroup.procs marker");
15670        let module_id = "reaped-module";
15671        let module = placement
15672            .module_path(module_id)
15673            .expect("create scratch module cgroup");
15674        let child = Command::new("true")
15675            .env("XDG_DATA_HOME", root.path())
15676            .env("XDG_RUNTIME_DIR", root.path())
15677            .env("XDG_CONFIG_HOME", root.path())
15678            .spawn()
15679            .expect("spawn short-lived child");
15680        let pid = child.id().expect("spawned child has pid");
15681        let mut child = SupervisedChild {
15682            child,
15683            exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15684            protocol: ModuleProtocol::Subc,
15685            module_id: module_id.to_string(),
15686            cgroup_placement: Some(placement),
15687            stdout_pump: None,
15688            stderr_pump: None,
15689            stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15690            spawned_at: tokio::time::Instant::now(),
15691            spawned_at_ms: 0,
15692            spawned_from: PathBuf::from("true"),
15693            launch_spec_digest: LaunchSpecDigest([0; 32]),
15694            spawned_file_identity: None,
15695            process_start_time: None,
15696            process_identity: None,
15697            pid,
15698            roster_guard: None,
15699            #[cfg(target_os = "macos")]
15700            privacy_exec: None,
15701            spawn_failure: None,
15702        };
15703
15704        child.wait().await.expect("reap short-lived child");
15705
15706        assert!(
15707            !module.exists(),
15708            "reaping the supervised child must remove its empty cgroup"
15709        );
15710    }
15711
15712    #[test]
15713    fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15714        let root = ScratchDir::new("supervisor-non-empty-cgroup");
15715        fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15716        let placement = subc_cgroup::prepare_at(&root)
15717            .expect("prepare scratch cgroup root")
15718            .expect("scratch root has a cgroup.procs marker");
15719        let module = placement
15720            .module_path("surviving-module")
15721            .expect("create scratch module cgroup");
15722        fs::write(module.join("surviving-process"), b"still present")
15723            .expect("make scratch cgroup non-empty");
15724        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15725
15726        remove_module_cgroup(&placement, "surviving-module");
15727
15728        let logs = crate::router::test_log::captured_logs(&logs);
15729        assert!(
15730            module.exists(),
15731            "failed removal must leave the cgroup intact"
15732        );
15733        assert!(
15734            logs.contains("could not remove module cgroup after process exit; continuing teardown")
15735                && logs.contains("surviving-module"),
15736            "best-effort removal must report the failure without returning it: {logs}"
15737        );
15738    }
15739
15740    #[test]
15741    fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15742        let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15743        let reason = SuperviseError::Spawn {
15744            program: PathBuf::from("/bin/true"),
15745            source: io::Error::from_raw_os_error(13),
15746            cgroup_path: Some(cgroup_path.clone()),
15747        }
15748        .to_string();
15749
15750        assert!(
15751            reason.contains(&cgroup_path.display().to_string()),
15752            "a pre_exec spawn failure must name the cgroup path: {reason}"
15753        );
15754    }
15755}
15756
15757#[cfg(test)]
15758mod spawn_subscriber_lag_tests {
15759    use super::*;
15760
15761    /// A subscriber whose connection stops draining is dropped once its frame
15762    /// channel fills. The client must learn that from a terminal Error frame
15763    /// after the frames already queued for it, not from a stream that simply
15764    /// goes quiet.
15765    #[tokio::test]
15766    async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15767        let feed = SpawnEventFeed::default();
15768        feed.configure_incarnation("lag-incarnation".to_string());
15769        // A one-slot connection queue that nobody reads until the emits are
15770        // done: the forwarder parks on it and the subscriber channel fills.
15771        let (tx, mut rx) = mpsc::channel(1);
15772        feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15773            .expect("subscribe");
15774        let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15775        for index in 0..emitted {
15776            feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15777            // Let the forwarder take what it can so the fill point is the
15778            // subscriber channel, not a scheduling accident.
15779            tokio::task::yield_now().await;
15780        }
15781        assert_eq!(
15782            feed.subscriber_count(),
15783            0,
15784            "the lagged subscriber must be removed"
15785        );
15786
15787        let mut data = Vec::new();
15788        let mut last = None;
15789        loop {
15790            let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15791                .await
15792                .expect("the forwarder must finish once the subscriber is dropped");
15793            let Some(outbound) = next else { break };
15794            let frame = outbound.frame;
15795            if frame.header.ty == FrameType::StreamData {
15796                assert!(last.is_none(), "no data may follow the terminal frame");
15797                let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15798                data.push(event.cursor.seq);
15799            } else {
15800                assert!(last.is_none(), "exactly one terminal frame");
15801                last = Some(frame);
15802            }
15803        }
15804        assert!(!data.is_empty(), "queued frames drain before the terminal");
15805        for pair in data.windows(2) {
15806            assert_eq!(
15807                pair[1],
15808                pair[0] + 1,
15809                "queued frames arrive dense and in order"
15810            );
15811        }
15812        let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15813        assert_eq!(terminal.header.ty, FrameType::Error);
15814        assert_eq!(terminal.header.corr, 7);
15815        let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15816        assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15817        let detail = body.detail.expect("lagged error carries detail");
15818        assert_eq!(
15819            detail["first_undelivered_cursor"]["seq"],
15820            data.last().unwrap() + 1,
15821            "the named cursor is the first event the subscriber did not receive"
15822        );
15823        assert_eq!(
15824            detail["first_undelivered_cursor"]["daemon_incarnation"],
15825            "lag-incarnation"
15826        );
15827    }
15828}
15829
15830#[cfg(test)]
15831mod terminal_history_read_concurrency_tests {
15832    use super::*;
15833    use crate::terminal_journal::read_pause;
15834    use cortexkit_test_support::ScratchDir;
15835    use std::sync::mpsc as std_mpsc;
15836
15837    fn journaled_ring(
15838        journal: &Arc<crate::terminal_journal::TerminalJournal>,
15839    ) -> Arc<Mutex<TerminalRing>> {
15840        Arc::new(Mutex::new(
15841            TerminalRing::new(TerminalRingConfig::default(), 1)
15842                .with_journal(Some(Arc::clone(journal))),
15843        ))
15844    }
15845
15846    fn crash(at_ms: u64) -> ExitReport {
15847        ExitReport {
15848            kind: ExitKind::Crash,
15849            code: Some(1),
15850            signal: None,
15851            at_ms,
15852        }
15853    }
15854
15855    /// Record an exit on another thread and report whether it finished within
15856    /// `bound`. The recorder thread is left running if it did not.
15857    fn record_within(
15858        module_id: &'static str,
15859        ring: &Arc<Mutex<TerminalRing>>,
15860        at_ms: u64,
15861        bound: Duration,
15862    ) -> bool {
15863        let ring = Arc::clone(ring);
15864        let (done, done_rx) = std_mpsc::channel();
15865        std::thread::spawn(move || {
15866            record_terminal(
15867                module_id,
15868                &ring,
15869                &SpawnEventFeed::default(),
15870                &crash(at_ms),
15871                TerminalDisposition::Restarting,
15872            );
15873            let _ = done.send(());
15874        });
15875        done_rx.recv_timeout(bound).is_ok()
15876    }
15877
15878    /// A history read in progress must not hold the journal writer (which every
15879    /// module's exit recording needs) or the module's own ring. Exits recorded
15880    /// while the read is paused complete promptly; the paused read answers as of
15881    /// the moment it started, and the next read has each exit exactly once.
15882    #[test]
15883    fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15884        let dir = ScratchDir::new("terminal-history-concurrent-read");
15885        let path = dir.join("terminals.jsonl");
15886        let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15887            path.clone(),
15888            "daemon".into(),
15889        ));
15890        let reader_ring = journaled_ring(&journal);
15891        let other_ring = journaled_ring(&journal);
15892        assert!(record_within(
15893            "reader-module",
15894            &reader_ring,
15895            10,
15896            Duration::from_secs(5)
15897        ));
15898
15899        let (started, release) = read_pause::install(&path);
15900        let reading = {
15901            let ring = Arc::clone(&reader_ring);
15902            std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15903        };
15904        started
15905            .recv_timeout(Duration::from_secs(5))
15906            .expect("the history read reached its pause");
15907
15908        let bound = Duration::from_secs(1);
15909        assert!(
15910            record_within("other-module", &other_ring, 20, bound),
15911            "another module's exit waited on a history read (journal writer held)"
15912        );
15913        assert!(
15914            record_within("reader-module", &reader_ring, 30, bound),
15915            "the read module's own exit waited on its history read (ring held)"
15916        );
15917
15918        drop(release);
15919        let paused = reading.join().unwrap();
15920        assert_eq!(
15921            paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15922            vec![10],
15923            "an exit recorded after the read began lands in neither half of it"
15924        );
15925        assert_eq!(paused.journal_skipped_lines, 0);
15926        assert_eq!(paused.journal_read_errors, 0);
15927
15928        let after = durable_terminal_history_of(&reader_ring, "reader-module");
15929        assert_eq!(
15930            after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15931            vec![10, 30],
15932            "the next read merges ring and journal with no duplicate"
15933        );
15934        assert_eq!(after.journal_skipped_lines, 0);
15935    }
15936}
15937
15938/// What a restart does with the exited process's output readers. These drive
15939/// the same settle functions the supervisor calls, with a reader the test
15940/// holds, so a reader that has not been scheduled by the bound is a controlled
15941/// input rather than something only a loaded machine produces.
15942#[cfg(test)]
15943mod stderr_settle_tests {
15944    use std::{
15945        future::Future,
15946        io,
15947        pin::Pin,
15948        sync::{Arc, Mutex},
15949        task::{Context, Poll},
15950        time::Duration,
15951    };
15952
15953    use tokio::{
15954        io::{AsyncRead, ReadBuf},
15955        sync::oneshot,
15956        time::Instant,
15957    };
15958
15959    use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15960    use crate::stderr_tail::{
15961        pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15962        ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15963    };
15964
15965    const BOUND: Duration = Duration::from_millis(250);
15966
15967    /// Yields `before`, then stays pending until the gate is released, then
15968    /// yields `after` and reaches EOF. The bytes after the gate were written
15969    /// by a process that has already exited; only the reader is behind.
15970    struct HeldReader {
15971        before: Option<Vec<u8>>,
15972        gate: Option<oneshot::Receiver<()>>,
15973        after: io::Cursor<Vec<u8>>,
15974    }
15975
15976    impl AsyncRead for HeldReader {
15977        fn poll_read(
15978            mut self: Pin<&mut Self>,
15979            cx: &mut Context<'_>,
15980            buf: &mut ReadBuf<'_>,
15981        ) -> Poll<io::Result<()>> {
15982            if let Some(bytes) = self.before.take() {
15983                buf.put_slice(&bytes);
15984                return Poll::Ready(Ok(()));
15985            }
15986            if let Some(gate) = self.gate.as_mut() {
15987                match Pin::new(gate).poll(cx) {
15988                    Poll::Pending => return Poll::Pending,
15989                    Poll::Ready(_) => self.gate = None,
15990                }
15991            }
15992            Pin::new(&mut self.after).poll_read(cx, buf)
15993        }
15994    }
15995
15996    struct DiscardSink;
15997
15998    impl OutputSink for DiscardSink {
15999        fn write_line(&mut self, _line: &[u8]) {}
16000    }
16001
16002    fn line(text: &str) -> TailEntry {
16003        TailEntry::Line {
16004            text: text.to_string(),
16005            truncated: false,
16006            at_ms: None,
16007        }
16008    }
16009
16010    fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16011        ring.lock().unwrap()
16012    }
16013
16014    /// Start a reader for a new process generation that delivers `before`
16015    /// immediately and `after` only once the returned sender fires (or is
16016    /// dropped).
16017    fn held_pump(
16018        ring: &Arc<Mutex<StderrRing>>,
16019        before: &str,
16020        after: &str,
16021    ) -> (StderrPump, oneshot::Sender<()>) {
16022        let generation = lock(ring).begin_process();
16023        let (release, gate) = oneshot::channel();
16024        let reader = HeldReader {
16025            before: Some(before.as_bytes().to_vec()),
16026            gate: Some(gate),
16027            after: io::Cursor::new(after.as_bytes().to_vec()),
16028        };
16029        let task = tokio::spawn(pump_stderr_to(
16030            reader,
16031            Arc::clone(ring),
16032            generation,
16033            DiscardSink,
16034        ));
16035        (StderrPump { task, generation }, release)
16036    }
16037
16038    async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16039        for _ in 0..1000 {
16040            if done(&lock(ring)) {
16041                return;
16042            }
16043            tokio::time::sleep(Duration::from_millis(1)).await;
16044        }
16045        panic!(
16046            "ring never reached the expected state: {:?}",
16047            lock(ring).snapshot(None, None)
16048        );
16049    }
16050
16051    #[tokio::test(start_paused = true)]
16052    async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16053        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16054        let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16055
16056        settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16057        let before_release = lock(&ring).snapshot(None, None);
16058        assert!(
16059            matches!(before_release.capture, CaptureState::Incomplete { .. }),
16060            "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16061        );
16062
16063        // The restart: the next process starts and writes before the old
16064        // reader catches up.
16065        let next = lock(&ring).begin_process();
16066        lock(&ring).push_line_from(next, "next process booting");
16067        release.send(()).unwrap();
16068        wait_until(&ring, |ring| {
16069            ring.snapshot(None, None).capture == CaptureState::Captured
16070        })
16071        .await;
16072
16073        assert_eq!(
16074            untimed(lock(&ring).snapshot(None, None).entries),
16075            vec![
16076                line("booting"),
16077                line("config error: missing storage"),
16078                TailEntry::ProcessStart,
16079                line("next process booting"),
16080            ],
16081            "the crash's last line must survive a slow reader and stay in the crashed process's section"
16082        );
16083    }
16084
16085    #[tokio::test(start_paused = true)]
16086    async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16087    ) {
16088        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16089        // `_held` is never fired: a descendant keeps the pipe open for the
16090        // whole test.
16091        let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16092
16093        let started = Instant::now();
16094        settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16095        assert_eq!(
16096            started.elapsed(),
16097            BOUND,
16098            "the restart must wait exactly the bound for a pipe that stays open, no longer"
16099        );
16100
16101        let next = lock(&ring).begin_process();
16102        lock(&ring).push_line_from(next, "next process booting");
16103        tokio::time::sleep(Duration::from_secs(60)).await;
16104
16105        let snapshot = lock(&ring).snapshot(None, None);
16106        match &snapshot.capture {
16107            CaptureState::Incomplete { reason } => assert!(
16108                reason.contains("had not reached EOF") && reason.contains("250ms"),
16109                "the reason must say what is missing and after how long: {reason}"
16110            ),
16111            other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16112        }
16113        assert_eq!(
16114            untimed(snapshot.entries),
16115            vec![
16116                line("parent exiting"),
16117                TailEntry::ProcessStart,
16118                line("next process booting"),
16119            ]
16120        );
16121    }
16122
16123    #[tokio::test(start_paused = true)]
16124    async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16125        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16126        let (pump, release) = held_pump(&ring, "one\n", "two\n");
16127        release.send(()).unwrap();
16128
16129        settle_stderr_pump("clean", &ring, pump, BOUND).await;
16130
16131        let snapshot = lock(&ring).snapshot(None, None);
16132        assert_eq!(snapshot.capture, CaptureState::Captured);
16133        assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16134    }
16135
16136    #[tokio::test(start_paused = true)]
16137    async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16138        const BURST: usize = 150;
16139        let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16140        let path = capture.join("two-pipe.stderr.log");
16141        let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16142        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16143        let generation = lock(&ring).begin_process();
16144        let stdout = (0..BURST)
16145            .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16146            .collect::<String>();
16147        let stderr = (0..BURST)
16148            .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16149            .collect::<String>();
16150        // Model a child that has written both bursts and exited. Hold the stdout
16151        // reader after one line and a partial prefix, as if pipe readiness or
16152        // capture-file writes had delayed it. The final line also needs EOF to
16153        // flush: neither complete nor partial buffered output may be discarded.
16154        let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16155        let (release, gate) = oneshot::channel();
16156        let reader = HeldReader {
16157            before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16158            gate: Some(gate),
16159            after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16160        };
16161        let (finished, completion) = oneshot::channel();
16162        let stdout_sink = sink.clone();
16163        let pump = tokio::spawn(async move {
16164            pump_stdout_to(reader, stdout_sink).await;
16165            finished.send(()).unwrap();
16166        });
16167        pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16168
16169        let started = Instant::now();
16170        settle_stdout_pump("two-pipe", pump, BOUND).await;
16171        assert_eq!(
16172            started.elapsed(),
16173            BOUND,
16174            "a late reader must not hold up restart"
16175        );
16176        assert_eq!(
16177            std::fs::read_to_string(&path).unwrap().lines().count(),
16178            BURST + 1
16179        );
16180
16181        release
16182            .send(())
16183            .expect("stdout reader must survive the exit drain bound");
16184        tokio::time::timeout(Duration::from_secs(1), completion)
16185            .await
16186            .expect("released stdout reader must reach EOF")
16187            .expect("stdout capture must finish at EOF, not be cancelled");
16188        let contents = std::fs::read_to_string(&path).unwrap();
16189        let mut lines = contents
16190            .lines()
16191            .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16192            .collect::<Vec<_>>();
16193        let mut expected = (0..BURST)
16194            .flat_map(|i| {
16195                [
16196                    format!("out-{i:04}-{}", "o".repeat(64)),
16197                    format!("err-{i:04}-{}", "e".repeat(64)),
16198                ]
16199            })
16200            .collect::<Vec<_>>();
16201        lines.sort_unstable();
16202        expected.sort_unstable();
16203        assert_eq!(
16204            lines, expected,
16205            "both pipes must retain every complete line exactly once"
16206        );
16207    }
16208}
16209
16210/// Containment of a module's process tree (issue #109).
16211///
16212/// The behaviour these defend against is a module helper surviving its module:
16213/// on a real machine the Synapse embedding module's CUDA worker holds ~2.2 GB of
16214/// VRAM, so a leaked grandchild is a leaked GPU allocation, and a day of restarts
16215/// compounds it.
16216///
16217/// They run against the SUPERVISOR rather than the job-object crate because the
16218/// claim is about teardown: a crate-level test proves a job can reap a tree, not
16219/// that the daemon's drain path reaches it.
16220///
16221/// Windows-only, like the mechanism. On Unix this arm compiles out; the cgroup
16222/// lane there is a separate containment path with its own tests.
16223#[cfg(all(test, windows))]
16224mod job_containment_tests {
16225    use super::*;
16226    use cortexkit_test_support::ScratchDir;
16227    use std::{
16228        path::{Path, PathBuf},
16229        sync::{Arc, Mutex},
16230        time::{Duration, Instant},
16231    };
16232
16233    /// The stub, expected beside this test executable.
16234    ///
16235    /// The existence check is here for the reason its twin at `fake_aft_stub_path`
16236    /// documents: `--lib` does not build `[[bin]]` targets, and a bare spawn
16237    /// failure then reads as a broken test rather than an unbuilt dependency.
16238    fn stub_path() -> PathBuf {
16239        let mut path = std::env::current_exe().expect("current_exe available in tests");
16240        path.pop();
16241        path.pop();
16242        path.push("fake-aft-stub.exe");
16243        assert!(
16244            path.exists(),
16245            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16246             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16247            path.display()
16248        );
16249        path
16250    }
16251
16252    /// Poll for the grandchild pid the stub records, and parse it.
16253    fn read_grandchild_pid(path: &Path) -> u32 {
16254        let deadline = Instant::now() + Duration::from_secs(10);
16255        loop {
16256            if let Ok(contents) = std::fs::read_to_string(path) {
16257                if let Ok(pid) = contents.trim().parse() {
16258                    return pid;
16259                }
16260            }
16261            assert!(
16262                Instant::now() < deadline,
16263                "the stub never recorded a grandchild pid at {}",
16264                path.display()
16265            );
16266            std::thread::sleep(Duration::from_millis(10));
16267        }
16268    }
16269
16270    /// Everything one fixture run needs, so the two tests below differ in exactly
16271    /// one place: whether the child is contained.
16272    struct Fixture {
16273        _dir: ScratchDir,
16274        module_id: String,
16275        grandchild: u32,
16276        child: Option<SupervisedChild>,
16277        registry: Arc<Registry>,
16278        snapshot: Arc<Mutex<SupervisorSnapshot>>,
16279        terminal_ring: Arc<Mutex<TerminalRing>>,
16280        spawn_events: SpawnEventFeed,
16281    }
16282
16283    fn fixture(label: &str, module_id: &str) -> Fixture {
16284        let dir = ScratchDir::new(label);
16285        let pid_file = dir.join("grandchild.pid");
16286        let supervisor = Supervisor::new_for_test(
16287            Arc::new(Registry::default()),
16288            RestartPolicy::new(3, Duration::ZERO),
16289        );
16290        let runtime = supervisor.runtime_config();
16291        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16292        let spec = ModuleSpec {
16293            module_id: module_id.to_string(),
16294            program: stub_path(),
16295            // Zero args deliberately: a `--subc` argument would make the stub dial
16296            // a daemon that is not there, and the failure would land in the same
16297            // stderr ring this fixture exists to keep quiet.
16298            args: Vec::new(),
16299            env: vec![
16300                ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16301                (
16302                    "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16303                    pid_file.display().to_string(),
16304                ),
16305            ],
16306            reserved: false,
16307            reserved_prefixes: Vec::new(),
16308            protocol: ModuleProtocol::Subc,
16309            overlap: Default::default(),
16310        };
16311        let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16312            .expect("spawn the supervised fixture");
16313        let grandchild = read_grandchild_pid(&pid_file);
16314        Fixture {
16315            _dir: dir,
16316            module_id: module_id.to_string(),
16317            grandchild,
16318            child: Some(child),
16319            registry: Arc::new(Registry::default()),
16320            snapshot,
16321            terminal_ring: Arc::clone(&runtime.terminal_ring),
16322            spawn_events: SpawnEventFeed::default(),
16323        }
16324    }
16325
16326    impl Fixture {
16327        /// Drain through the supervisor's own teardown path.
16328        async fn drain(&mut self) {
16329            let child = self
16330                .child
16331                .take()
16332                .expect("the fixture child is still present");
16333            drain_child_to_state(
16334                &self.module_id,
16335                ModuleProtocol::Subc,
16336                // No forwarding table in this fixture, so nothing reaches the
16337                // child over a connection.
16338                StopNotice::NotSent,
16339                &self.registry,
16340                None,
16341                &self.snapshot,
16342                &self.terminal_ring,
16343                &self.spawn_events,
16344                child,
16345                Duration::from_millis(500),
16346                ModuleState::Stopped,
16347                Some(false),
16348            )
16349            .await
16350            .expect("drain the supervised fixture");
16351        }
16352    }
16353
16354    /// Teardown reaps the grandchild, not merely the direct child.
16355    ///
16356    /// This is the assertion the change exists for. Before containment the
16357    /// grandchild survived: it is a separate process, and `start_kill` is
16358    /// `TerminateProcess` scoped to one pid.
16359    #[tokio::test]
16360    async fn teardown_reaps_the_grandchild() {
16361        let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16362        let grandchild = fixture.grandchild;
16363
16364        assert!(
16365            subc_jobobject::process_exists(grandchild),
16366            "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16367        );
16368
16369        fixture.drain().await;
16370
16371        assert!(
16372            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16373            "grandchild {grandchild} outlived module teardown: the tree was not contained"
16374        );
16375    }
16376
16377    /// The mutation control: with containment withheld, the grandchild survives
16378    /// the same kill.
16379    ///
16380    /// This is the defect reproduction from #109 — a direct-child kill reaches
16381    /// one pid, and the grandchild is a different process. It spawns OUTSIDE the
16382    /// supervisor because `spawn_and_mark_running` now always contains on
16383    /// Windows, which is the point: there is no longer a path that spawns
16384    /// uncontained, so the control has to construct one.
16385    ///
16386    /// Its job is to keep `teardown_reaps_the_grandchild` honest. If the
16387    /// grandchild ever dies here, that test is passing for a reason unrelated to
16388    /// the job object and the containment claim is unproven.
16389    #[test]
16390    fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16391        let dir = ScratchDir::new("teardown-uncontained");
16392        let pid_file = dir.join("grandchild.pid");
16393        let mut child = std::process::Command::new(stub_path())
16394            .env("FAKE_AFT_NEVER_CONNECT", "1")
16395            .env(
16396                "FAKE_AFT_GRANDCHILD_PID_FILE",
16397                pid_file.display().to_string(),
16398            )
16399            .stdin(std::process::Stdio::null())
16400            .stdout(std::process::Stdio::null())
16401            .stderr(std::process::Stdio::null())
16402            .spawn()
16403            .expect("spawn the uncontained fixture");
16404        let grandchild = read_grandchild_pid(&pid_file);
16405
16406        // Exactly what the pre-fix teardown did: kill the direct child.
16407        child.kill().expect("kill the direct child");
16408        let _ = child.wait();
16409
16410        assert!(
16411            subc_jobobject::process_exists(grandchild),
16412            "grandchild {grandchild} died with the direct child, so this control no longer \
16413             distinguishes contained from uncontained teardown and the regression test is \
16414             passing vacuously"
16415        );
16416
16417        // The orphan this control demonstrates is the leak the fix prevents, so
16418        // the control must not leave one behind.
16419        kill_tree(grandchild);
16420    }
16421
16422    /// Crash durability: closing the containment handle reaps the tree with no
16423    /// teardown code running at all.
16424    ///
16425    /// This is the case `taskkill /T` cannot cover — a daemon that dies cannot
16426    /// call anything — and it is why containment is a kernel property of the
16427    /// handle rather than a step in the drain. Discovered by getting the
16428    /// mutation control wrong: clearing `job` to "disable" containment instead
16429    /// killed the tree, which is the guarantee, not a mistake.
16430    #[tokio::test]
16431    async fn dropping_containment_reaps_the_grandchild() {
16432        let mut fixture = fixture("drop-containment", "tree-drop");
16433        let grandchild = fixture.grandchild;
16434
16435        assert!(subc_jobobject::process_exists(grandchild));
16436
16437        // No `drain` call, no kill: dropping the handle is the entire mechanism.
16438        fixture.child.as_mut().expect("child present").job = None;
16439
16440        assert!(
16441            subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16442            "grandchild {grandchild} survived the containment handle closing, so a daemon \
16443             crash would leave the tree behind"
16444        );
16445    }
16446
16447    /// Kill a pid and its tree, then confirm it is gone.
16448    fn kill_tree(pid: u32) {
16449        let _ = std::process::Command::new("taskkill.exe")
16450            .args(["/PID", &pid.to_string(), "/T", "/F"])
16451            .stdin(std::process::Stdio::null())
16452            .stdout(std::process::Stdio::null())
16453            .stderr(std::process::Stdio::null())
16454            .status();
16455        assert!(
16456            subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16457            "could not clean up grandchild {pid}"
16458        );
16459    }
16460}
16461
16462#[cfg(test)]
16463mod privacy_trampoline_configuration_tests {
16464    #[tokio::test]
16465    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16466    async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16467        #[cfg(target_os = "macos")]
16468        {
16469            let supervisor = super::Supervisor::new(
16470                std::sync::Arc::new(crate::Registry::default()),
16471                super::RestartPolicy::default(),
16472            );
16473            let error = supervisor.spawn(spec()).unwrap_err();
16474            assert!(
16475                error
16476                    .to_string()
16477                    .contains("no privacy trampoline configured"),
16478                "{error}"
16479            );
16480        }
16481    }
16482
16483    #[tokio::test]
16484    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16485    async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16486        #[cfg(target_os = "macos")]
16487        {
16488            let supervisor = super::Supervisor::new(
16489                std::sync::Arc::new(crate::Registry::default()),
16490                super::RestartPolicy::default(),
16491            )
16492            .with_privacy_trampoline(std::env::current_exe().unwrap());
16493            let error = supervisor.spawn(spec()).unwrap_err();
16494            assert!(
16495                error
16496                    .to_string()
16497                    .contains("binary does not implement the privacy trampoline protocol"),
16498                "{error}"
16499            );
16500        }
16501    }
16502
16503    #[cfg(target_os = "macos")]
16504    fn spec() -> super::ModuleSpec {
16505        super::ModuleSpec {
16506            module_id: "privacy-configuration".into(),
16507            program: "/bin/sleep".into(),
16508            args: vec!["30".into()],
16509            env: vec![],
16510            reserved: false,
16511            reserved_prefixes: vec![],
16512            protocol: subc_control::ModuleProtocol::None,
16513            overlap: super::ModuleOverlap::Exclusive,
16514        }
16515    }
16516}
16517
16518#[cfg(test)]
16519mod privacy_exec_boundary_tests {
16520    #[cfg(target_os = "macos")]
16521    use super::*;
16522    #[cfg(target_os = "macos")]
16523    use std::{
16524        io::{Read, Write},
16525        net::{TcpListener, TcpStream},
16526    };
16527
16528    /// Unit-test-only pause at the actual early image read, not at a later
16529    /// status read. Production supervisors never inspect this environment key.
16530    #[cfg(target_os = "macos")]
16531    pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16532        if let Some((_, path)) = spec
16533            .env
16534            .iter()
16535            .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16536        {
16537            let mut barrier = TcpStream::connect(path).unwrap();
16538            barrier
16539                .set_read_timeout(Some(Duration::from_secs(30)))
16540                .unwrap();
16541            barrier.write_all(&pid.to_ne_bytes()).unwrap();
16542            let mut release = [0];
16543            barrier.read_exact(&mut release).unwrap();
16544            assert_eq!(&release, b"X");
16545        }
16546    }
16547
16548    #[cfg(target_os = "macos")]
16549    fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16550        ModuleSpec {
16551            module_id: "privacy-boundary".into(),
16552            program: program.into(),
16553            args: args.iter().map(|arg| (*arg).into()).collect(),
16554            env: vec![],
16555            reserved: false,
16556            reserved_prefixes: vec![],
16557            protocol: ModuleProtocol::None,
16558            overlap: ModuleOverlap::Exclusive,
16559        }
16560    }
16561
16562    #[cfg(target_os = "macos")]
16563    async fn accept(listener: TcpListener) -> TcpStream {
16564        // Socket readiness, not elapsed time, establishes both pause points.
16565        let listener = tokio::net::TcpListener::from_std({
16566            listener.set_nonblocking(true).unwrap();
16567            listener
16568        })
16569        .unwrap();
16570        let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16571            .await
16572            .unwrap()
16573            .unwrap();
16574        let stream = stream.into_std().unwrap();
16575        stream.set_nonblocking(false).unwrap();
16576        stream
16577            .set_read_timeout(Some(Duration::from_secs(30)))
16578            .unwrap();
16579        stream
16580    }
16581
16582    #[tokio::test]
16583    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16584    async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16585        #[cfg(target_os = "macos")]
16586        {
16587            let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16588            // Loopback sockets also work when the replay adapter's TMPDIR is
16589            // longer than Darwin's Unix-domain socket path limit.
16590            let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16591            let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16592            let record = root.join("live-children.json");
16593            let supervisor =
16594                Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16595                    .with_live_children_record(&record);
16596            let runtime = supervisor.runtime_config();
16597            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16598            let mut spec = spec("/bin/sleep", &["30"]);
16599            spec.env = vec![
16600                (
16601                    "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16602                    exec_listener.local_addr().unwrap().to_string(),
16603                ),
16604                (
16605                    "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16606                    sample_listener.local_addr().unwrap().to_string(),
16607                ),
16608            ];
16609            let spawn = tokio::task::spawn_blocking(move || {
16610                spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16611            });
16612            let mut sample = accept(sample_listener).await;
16613            let mut pid = [0; 4];
16614            sample.read_exact(&mut pid).unwrap();
16615            let pid = u32::from_ne_bytes(pid);
16616            let mut exec = accept(exec_listener).await;
16617            let mut ready = [0];
16618            exec.read_exact(&mut ready).unwrap();
16619            assert_eq!(&ready, b"R");
16620            // The early read is guaranteed to see a real, non-null trampoline
16621            // image: the fixture has reached its barrier and cannot exec yet.
16622            let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16623            assert_eq!(
16624                observe_spawned_image(pid).unwrap().executable,
16625                Some(trampoline)
16626            );
16627            sample.write_all(b"X").unwrap();
16628            let mut child = spawn.await.unwrap();
16629            let early = crate::live_children::read_record(&record).unwrap();
16630            assert_eq!(early.len(), 1);
16631            assert_eq!(early[0].pid, pid);
16632            assert_eq!(
16633                early[0].executable, None,
16634                "unconfirmed trampoline image entered the roster"
16635            );
16636            assert!(child.report_ready.get().is_none());
16637            // The barrier's duration is unrelated to the production five-second
16638            // exec budget. Start the test's confirmation budget upon release.
16639            child.privacy_exec.as_mut().unwrap().deadline =
16640                tokio::time::Instant::now() + Duration::from_secs(30);
16641            exec.write_all(b"X").unwrap();
16642            child.confirm_privacy_exec().await;
16643            assert_eq!(child.spawn_failure, None);
16644            assert!(child.report_ready.get().is_some());
16645            let confirmed = crate::live_children::read_record(&record).unwrap();
16646            let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16647            assert_ne!(module, trampoline);
16648            assert_eq!(confirmed[0].executable, Some(module.into()));
16649            child.start_kill().unwrap();
16650            child.wait().await.unwrap();
16651            child.release_roster();
16652        }
16653    }
16654
16655    #[tokio::test]
16656    #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16657    async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16658        #[cfg(target_os = "macos")]
16659        {
16660            let registry = Arc::new(Registry::default());
16661            let policy = RestartPolicy::new(0, Duration::ZERO);
16662            let supervisor = Supervisor::new_for_test(Arc::clone(&registry), policy);
16663            let runtime = supervisor.runtime_config();
16664            let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16665            let spec = spec("/bin/sh", &["-c", "exit 121"]);
16666            // Drive spawn and confirmation separately instead of starting the
16667            // monitor. WNOWAIT observes a real exit without consuming its status,
16668            // so confirmation's first try_wait must take the already-exited arm.
16669            let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16670            let pid = child.pid;
16671            tokio::task::spawn_blocking(move || {
16672                subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16673            })
16674            .await
16675            .unwrap()
16676            .unwrap();
16677            child.privacy_exec.as_mut().unwrap().deadline =
16678                tokio::time::Instant::now() + Duration::from_secs(30);
16679            let status = child.wait().await.unwrap();
16680            assert_eq!(status.code(), Some(121));
16681            assert!(child.privacy_exec.is_none());
16682            assert!(
16683                child.report_ready.get().is_none(),
16684                "an exited module must not publish a live pid"
16685            );
16686            let report = classify_reaped_child_exit(&snapshot, &child, &status);
16687            on_child_exit(
16688                &spec,
16689                policy,
16690                &registry,
16691                &snapshot,
16692                &runtime.terminal_ring,
16693                &runtime.spawn_events,
16694                &runtime.child_roster,
16695                report,
16696            )
16697            .await;
16698            let state = lock_snapshot(&snapshot).unwrap();
16699            assert_eq!(state.state, ModuleState::Failed);
16700            assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16701            assert_eq!(state.reported_pid(), None);
16702            drop(state);
16703            let history = runtime.terminal_ring.lock().unwrap().snapshot();
16704            assert_eq!(history.entries.len(), 1);
16705            let terminal = &history.entries[0];
16706            assert_eq!(terminal.exit_code, Some(121));
16707            assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16708            assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16709            assert_eq!(
16710                terminal.disposition_detail.as_deref(),
16711                Some(policy.budget_exhausted_detail().as_str()),
16712                "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16713            );
16714            assert_eq!(child.spawn_failure, None);
16715            child.release_roster();
16716        }
16717    }
16718}
16719
16720/// The daemon's real spawn path hands a subc-wire child its launch nonce on
16721/// descriptor 3, without an environment copy. The shell records the nonce
16722/// and its environment after exec so these tests observe the real handover.
16723#[cfg(all(test, unix))]
16724mod launch_nonce_descriptor_tests {
16725    use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16726    use crate::stderr_tail::{StderrRing, StderrTailConfig};
16727    use cortexkit_test_support::ScratchDir;
16728    use std::{
16729        path::PathBuf,
16730        sync::{Arc, Mutex},
16731        time::{Duration, Instant},
16732    };
16733
16734    async fn probe(role: super::SpawnRole) {
16735        let scratch = ScratchDir::new("launch-nonce-descriptor");
16736        let fd_copy = scratch.join("from-descriptor");
16737        let env_copy = scratch.join("environment");
16738        let script = format!(
16739            "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16740            fd = fd_copy.display(), env = env_copy.display(),
16741        );
16742        let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16743        let spec = ModuleSpec {
16744            module_id: "nonce-descriptor-probe".to_string(),
16745            program: PathBuf::from("/bin/sh"),
16746            args: vec!["-c".to_string(), script],
16747            env: vec![
16748                xdg("XDG_DATA_HOME"),
16749                xdg("XDG_RUNTIME_DIR"),
16750                xdg("XDG_CONFIG_HOME"),
16751                ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16752            ],
16753            reserved: true,
16754            reserved_prefixes: Vec::new(),
16755            protocol: ModuleProtocol::Subc,
16756            overlap: Default::default(),
16757        };
16758        let handle = SupervisorHandle::new();
16759        let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16760        let roster = ChildRoster::default();
16761        #[cfg(target_os = "macos")]
16762        {
16763            let path = super::test_privacy_trampoline();
16764            roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16765        }
16766        let child = super::spawn_child_in_slot(
16767            &spec,
16768            None,
16769            Some(&handle),
16770            &ring,
16771            None,
16772            &roster,
16773            #[cfg(target_os = "linux")]
16774            None,
16775            role,
16776            matches!(role, super::SpawnRole::SwapCandidate),
16777        )
16778        .expect("spawn probe");
16779        let deadline = Instant::now() + Duration::from_secs(10);
16780        while !(fd_copy.exists() && env_copy.exists()) {
16781            assert!(Instant::now() < deadline, "probe never wrote its copies");
16782            tokio::time::sleep(Duration::from_millis(20)).await;
16783        }
16784        let nonce = std::fs::read_to_string(fd_copy).unwrap();
16785        assert!(!nonce.is_empty());
16786        let environment = std::fs::read_to_string(env_copy).unwrap();
16787        assert!(environment
16788            .lines()
16789            .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16790        let copy = environment
16791            .lines()
16792            .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16793        assert_eq!(
16794            copy, None,
16795            "Unix children must never receive the environment nonce"
16796        );
16797        if matches!(role, super::SpawnRole::Plain) {
16798            assert_eq!(
16799                handle.spawn_nonce(&spec.module_id).as_deref(),
16800                Some(nonce.as_str())
16801            );
16802        }
16803        drop(child);
16804    }
16805
16806    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16807    async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16808        probe(super::SpawnRole::Plain).await;
16809    }
16810
16811    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16812    async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16813        probe(super::SpawnRole::SwapCandidate).await;
16814    }
16815}
16816
16817#[cfg(all(test, target_os = "linux"))]
16818mod cgroup_containment_tests {
16819    use super::*;
16820    use cortexkit_test_support::ScratchDir;
16821
16822    fn running(pid: u32) -> bool {
16823        // An orphan can remain a zombie until the container init reaps it.
16824        std::fs::read_to_string(format!("/proc/{pid}/stat"))
16825            .ok()
16826            .and_then(|stat| {
16827                stat.rsplit_once(") ")
16828                    .map(|(_, rest)| rest.starts_with('Z'))
16829            })
16830            .is_some_and(|zombie| !zombie)
16831    }
16832
16833    #[tokio::test]
16834    async fn linux_teardown_reaps_the_grandchild() {
16835        teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16836    }
16837
16838    #[tokio::test]
16839    async fn linux_shutdown_straggler_reaps_the_grandchild() {
16840        teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16841    }
16842
16843    async fn teardown_tree(test_name: &str, shutdown: bool) {
16844        let dir = ScratchDir::new(test_name);
16845        let root = PathBuf::from(format!(
16846            "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16847            std::process::id(),
16848            unix_ms_now()
16849        ));
16850        if let Err(error) = std::fs::create_dir(&root) {
16851            assert!(
16852                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16853                "required cgroup test cannot execute: {error}"
16854            );
16855            eprintln!(
16856                "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16857                root.display()
16858            );
16859            return;
16860        }
16861        let placement = subc_cgroup::prepare_at(&root)
16862            .expect("prepare isolated kernel cgroup")
16863            .expect("isolated cgroup is delegated");
16864        let module_id = "tree-teardown";
16865        let module = placement
16866            .module_path(module_id)
16867            .expect("create isolated module cgroup");
16868        if !module.join("cgroup.kill").exists() {
16869            std::fs::remove_dir(&module).unwrap();
16870            std::fs::remove_dir(root.join("subc-modules")).unwrap();
16871            std::fs::remove_dir(&root).unwrap();
16872            assert!(
16873                std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16874                "required cgroup.kill interface unavailable"
16875            );
16876            eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16877            return;
16878        }
16879        let supervisor = Supervisor::new_for_test(
16880            Arc::new(Registry::default()),
16881            RestartPolicy::new(3, Duration::ZERO),
16882        )
16883        .with_cgroup_placement(Some(placement));
16884        let mut runtime = supervisor.runtime_config();
16885        runtime.child_roster = runtime
16886            .child_roster
16887            .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16888        let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16889        let pid_file = dir.join("grandchild.pid");
16890        let spec = ModuleSpec {
16891            module_id: module_id.to_string(),
16892            program: PathBuf::from("/bin/sh"),
16893            args: vec![
16894                "-c".into(),
16895                "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16896                "fixture".into(),
16897                pid_file.display().to_string(),
16898            ],
16899            env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16900                .into_iter()
16901                .map(|key| (key.to_string(), dir.display().to_string()))
16902                .collect(),
16903            reserved: false,
16904            reserved_prefixes: Vec::new(),
16905            protocol: ModuleProtocol::None,
16906            overlap: Default::default(),
16907        };
16908        let child =
16909            spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16910        let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16911        let grandchild: u32 = loop {
16912            if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16913                if let Ok(pid) = contents.trim().parse() {
16914                    break pid;
16915                }
16916            }
16917            assert!(
16918                tokio::time::Instant::now() < deadline,
16919                "grandchild pid was not recorded"
16920            );
16921            tokio::time::sleep(Duration::from_millis(10)).await;
16922        };
16923        assert!(
16924            running(grandchild),
16925            "grandchild must be alive before teardown"
16926        );
16927        if shutdown {
16928            let mut child = child;
16929            crate::child_roster::end_children_for_daemon_shutdown(
16930                &runtime.child_roster,
16931                false,
16932                std::future::pending(),
16933            )
16934            .await;
16935            child.wait().await.expect("reap shutdown straggler");
16936        } else {
16937            drain_child_to_state(
16938                module_id,
16939                ModuleProtocol::None,
16940                StopNotice::NotSent,
16941                &Registry::default(),
16942                None,
16943                &snapshot,
16944                &runtime.terminal_ring,
16945                &SpawnEventFeed::default(),
16946                child,
16947                Duration::from_millis(100),
16948                ModuleState::Stopped,
16949                Some(false),
16950            )
16951            .await
16952            .expect("real supervisor teardown");
16953        }
16954        let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16955        while running(grandchild) && tokio::time::Instant::now() < deadline {
16956            tokio::time::sleep(Duration::from_millis(10)).await;
16957        }
16958        let survived = running(grandchild);
16959        // Kill a surviving grandchild so a failed test does not leave it behind.
16960        if survived {
16961            let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16962            let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16963            tokio::time::sleep(Duration::from_millis(100)).await;
16964        }
16965        if module.exists() {
16966            std::fs::remove_dir(&module).expect("remove empty module cgroup");
16967        }
16968        std::fs::remove_dir(root.join("subc-modules")).unwrap();
16969        std::fs::remove_dir(&root).unwrap();
16970        assert!(
16971            !survived,
16972            "grandchild {grandchild} outlived module teardown"
16973        );
16974        eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16975    }
16976}
16977
16978/// A slow live-children record write on the restart path must not stop the
16979/// daemon from answering the replacement module's authentication handshake.
16980#[cfg(all(test, unix))]
16981mod record_write_stall_tests {
16982    use super::*;
16983    use std::sync::atomic::{AtomicBool, Ordering};
16984    use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16985
16986    /// `subc-client-rs` gives the whole handshake this long before the module
16987    /// exits with `Timeout { stage: ServerProof }`.
16988    const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16989    /// How long the replacement's admission write is held: longer than the
16990    /// module's deadline, standing in for the delay a busy disk can add
16991    /// across the two syncs in that write.
16992    const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
16993
16994    /// A module that runs `/bin/sleep` directly, not through `/bin/sh -c`.
16995    /// On macOS every module is launched through the privacy trampoline, and
16996    /// a module that is not a `#!` script is admitted only if its running
16997    /// image is the configured program. With `/bin/sh -c "exec sleep 600"`
16998    /// the shell soon replaces itself with `/bin/sleep`, which does not match
16999    /// the configured `/bin/sh`. The image is read only after the spawn's
17000    /// live-children record write, and here that write lasted long enough for
17001    /// the shell to have exec'd every time, so each process was refused and
17002    /// killed right after spawn, and the supervisor crash-looped it with
17003    /// backoff until its restart budget ran out. The restart this test
17004    /// measures would then never produce a module at all.
17005    fn sleeping_spec() -> ModuleSpec {
17006        ModuleSpec {
17007            module_id: "record-write-stall".into(),
17008            program: "/bin/sleep".into(),
17009            args: vec!["600".into()],
17010            env: vec![],
17011            reserved: false,
17012            reserved_prefixes: vec![],
17013            protocol: ModuleProtocol::None,
17014            overlap: Default::default(),
17015        }
17016    }
17017
17018    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17019    async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17020        let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17021        let supervisor =
17022            Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17023                .with_live_children_record(home.join("live-children.json"));
17024        let module = supervisor.spawn(sleeping_spec()).unwrap();
17025        // `reported_pid` names a process only once it is confirmed as the
17026        // module (on macOS, once the trampoline's image check has passed), so
17027        // a process the daemon is about to refuse and kill never counts as
17028        // the running module, here or in the final wait below.
17029        let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17030            loop {
17031                if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17032                    .unwrap()
17033                    .reported_pid()
17034                {
17035                    return pid;
17036                }
17037                tokio::time::sleep(Duration::from_millis(10)).await;
17038            }
17039        })
17040        .await
17041        .expect("the first process was admitted as the module");
17042
17043        // The daemon's listener, served by this runtime's workers like the
17044        // supervisor's own tasks.
17045        let key = vec![0x42; 32];
17046        let daemon_id = [0x24; 16];
17047        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17048        let port = listener.local_addr().unwrap().port();
17049        let server = tokio::spawn(crate::server::serve_listener(
17050            listener,
17051            Arc::new(crate::router::Router::with_default_self_handler()),
17052            crate::server::ServerAuth::with_limits(
17053                key.clone(),
17054                daemon_id,
17055                "test",
17056                crate::server::DEFAULT_AUTH_DEADLINE,
17057                4,
17058            ),
17059        ));
17060
17061        // Hold the write that admits the replacement, once, and say when it
17062        // has started.
17063        let (held_tx, held_rx) = std::sync::mpsc::channel();
17064        let armed = AtomicBool::new(true);
17065        supervisor
17066            .child_roster
17067            .set_record_write_hook_for_test(move |live| {
17068                let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17069                if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17070                    let _ = held_tx.send(());
17071                    std::thread::sleep(HELD_RECORD_WRITE);
17072                }
17073            });
17074
17075        // The replacement module, as a separate process would be: its own
17076        // thread and runtime, so only the daemon's runtime can be stalled.
17077        let module_process = std::thread::spawn(move || {
17078            held_rx
17079                .recv_timeout(Duration::from_secs(30))
17080                .expect("the restart reached the replacement's record write");
17081            let conn = ConnectionInfo {
17082                schema: SCHEMA_VERSION,
17083                wire_version: None,
17084                endpoints: vec![Endpoint {
17085                    host: "127.0.0.1".to_owned(),
17086                    port,
17087                }],
17088                key,
17089                daemon_id,
17090                pid: std::process::id(),
17091                daemon_ver: "test".to_owned(),
17092            };
17093            tokio::runtime::Builder::new_current_thread()
17094                .enable_all()
17095                .build()
17096                .unwrap()
17097                .block_on(async {
17098                    let started = Instant::now();
17099                    let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17100                        .await
17101                        .unwrap();
17102                    let result =
17103                        authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17104                    (result, started.elapsed())
17105                })
17106        });
17107
17108        module.restart(Some(0)).await.unwrap();
17109        let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17110            .await
17111            .unwrap();
17112        assert!(
17113            auth.is_ok(),
17114            "the daemon must answer a module's handshake while a restart waits on the \
17115             live-children record write; got {auth:?} after {elapsed:?}"
17116        );
17117
17118        tokio::time::timeout(Duration::from_secs(20), async {
17119            while lock_snapshot(&module.inner.snapshot)
17120                .unwrap()
17121                .reported_pid()
17122                .is_none_or(|pid| pid == old_pid)
17123            {
17124                tokio::time::sleep(Duration::from_millis(10)).await;
17125            }
17126        })
17127        .await
17128        .expect("the restart published its replacement");
17129        module.stop().await.unwrap();
17130        server.abort();
17131    }
17132}